diff options
Diffstat (limited to 'test/soak.py')
| -rw-r--r-- | test/soak.py | 1318 |
1 files changed, 1318 insertions, 0 deletions
diff --git a/test/soak.py b/test/soak.py new file mode 100644 index 00000000..f75371ef --- /dev/null +++ b/test/soak.py @@ -0,0 +1,1318 @@ +#!/usr/bin/env python3 +"""A soak of replica clients: what a person does in them, through each link, +measured (docs/next-daemon.md, the replica and GUI replica notes). + + python3 -B test/soak.py local [--minutes 2] [--rtts 0,30,80,207] [--jitter 10] + [--seed 1] [--clients tty,win] [--pardes P] [--gui G] [--trace] + python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B] + [--pardes P] [--gui G] [--seed 1] [--trace] + +`local`: a session on this machine (`pardes --detach=soak <a directory>`, the +shape of the user's: a directory pane, then two shells), and at each RTT its +clients through test/delay_relay.py (with jitter), far as through ssh +(PARDES_REPLICA_FAR=1: the blobs by hash, the stream in zstd). One cell an +RTT, `--minutes` each. `box`: the session on the FreeBSD box under +/root/pardes-test (`--remote-bin`, by default bin/pardes-hunt there, run in +run-hunt and home-hunt), the clients here through ssh, as the user's window +was; its 9P through an `ssh -L` of the session's socket. + +The clients: terminals (`pardes --attach`, in a pty here whose screen pyte +keeps, frame requests answered) and windows (`pardes-gui --attach`, SDL's +offscreen driver, keys and the mouse through its test feed on a pty, so it +can be resized). Each runs a seeded monkey of a person's actions: moves of +every kind (keys, clicks, jumps, search), scrolls (keys, the wheel), edits, +undo and save, Look at files and directories, the Panes picker, Find, new +terminals with programs (ls, a long output, less, vi, top, Ctrl-C), panes and +columns opened, moved and closed, the pointer resting and moving, pastes, +idle gaps. Between them, at random: a client resized, a client detached and +another attached, an Upgrade, a 9P script's edit. `--seed` makes a run +again (the timing differs, the actions do not). + +Measured (a row a client, then the session's): + - rejoins: links past the first (PARDES_REPLICA_STATS writes one a link) + and `joined again` said by it; checks (PARDES_REPLICA_CHECK) that differ; + - exits nobody asked for, a signal or a non-zero code, the session's death, + a panic or `crashes` line; + - freezes: a quiet point where a client gives no fresh check within 8 s, + and a terminal client's key with no output within 2 s while it was not + held; the session's worst boundary (its stats at its end); + - the view: at each quiet point, a terminal client's cursor and screen + (pyte) against what the session renders for its view (/screen read right + after its own input made its view the live one), and the panes on it; + - blink: visible rollbacks (`visible`), mispredicted inputs, a terminal + client's full clears, a minute; + - keystroke to output (terminal) and to a frame (window, PARDES_TEST_LATENCY) + p50 / p99. + +Prints a table, then each breach, and exits 1 if there is any. Everything +started is ended by pid (and found by its environment); the run's files are +kept under $TMPDIR/soak-* (`--keep`) or removed. +""" +import argparse +import fcntl +import json +import os +import random +import re +import shutil +import signal +import socket +import statistics +import struct +import subprocess +import sys +import tempfile +import termios +import threading +import time +import tty as ttymod +from pathlib import Path + +import pyte + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from ninep import Client # noqa: E402 +from daemon import processes, alive, wait_for, checks # noqa: E402 + +BOX = '[email protected]' +SSH = ['/usr/bin/ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes', + '-o', 'UserKnownHostsFile=/home/goblin/.ssh/known_hosts', '-i', '/home/goblin/.ssh/id_ed25519'] +BOX_T = '/root/pardes-test' +NAME = 'soak' +ESC = b'\x1b' + + +def now(): + return time.monotonic() + + +def pct(xs, p): + if not xs: + return None + xs = sorted(xs) + return xs[min(len(xs) - 1, len(xs) * p // 100)] + + +# ---- the fixture: a directory such as a person works in ---- + +WORDS = ('alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa ' + 'quebec romeo sierra tango uniform victor whiskey xray yankee zulu').split() + + +def fixture(at, rng): + at.mkdir(parents=True, exist_ok=True) + (at / 'notes.txt').write_text(''.join(f'line {i} of the notes: {rng.choice(WORDS)} {rng.choice(WORDS)} see code.zig:{i % 90 + 1}\n' + for i in range(300))) + (at / 'code.zig').write_text(''.join(f'pub fn f{i}(x: u32) u32 {{\n return x * {i} + {rng.randrange(1000)}; // {rng.choice(WORDS)}\n}}\n\n' + for i in range(120))) + (at / 'big.txt').write_text(''.join(f'{i:06d} {" ".join(rng.choice(WORDS) for _ in range(9))}\n' for i in range(8000))) + (at / 'README').write_text('A directory to soak in.\nnotes.txt code.zig big.txt sub/\nsub/inner.txt:3\n') + sub = at / 'sub' + sub.mkdir(exist_ok=True) + (sub / 'inner.txt').write_text(''.join(f'inner {i} {rng.choice(WORDS)}\n' for i in range(60))) + (sub / 'more.md').write_text('# More\n\n' + ''.join(f'- item {i} {rng.choice(WORDS)}\n' for i in range(40))) + + +# ---- terminals ---- + +class Term: + """A terminal: a pty whose output is kept, timed and fed to pyte, its + frame requests (CSI 5 n) answered as a terminal emulator does. An Esc is + written alone and nothing follows it for 80 ms, an answer included: the + client's parser takes an Esc and what follows within 50 ms as Alt.""" + + def __init__(self, rows, cols): + self.master, self.slave = os.openpty() + self.rows, self.cols = rows, cols + fcntl.ioctl(self.slave, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + self.lock = threading.Lock() + self.wlock = threading.Lock() + self.screen = pyte.Screen(cols, rows) + self.stream = pyte.ByteStream(self.screen) + self.out_bytes = 0 + self.last_out = 0.0 + self.chunks = [] # (time, bytes) of the last minute or so, for latencies + self.clears = 0 + self.joined_again = 0 + self.tail = b'' + self.esc_until = 0.0 + self.closing = False + self.log = None + self.reader = threading.Thread(target=self.drain, daemon=True) + self.reader.start() + + def drain(self): + import select + while not self.closing: + try: + if not select.select([self.master], [], [], 0.05)[0]: + continue + got = os.read(self.master, 65536) + except OSError: + return + if not got: + return + t = now() + carry, self.tail = self.tail, (self.tail + got)[-11:] + seen = carry[-3:] + got + with self.lock: + self.out_bytes += len(got) + self.last_out = t + self.chunks.append((t, len(got))) + if len(self.chunks) > 4096: + del self.chunks[:2048] + # The carry is shorter than either word: a match holds a new byte. + self.clears += seen.count(b'\x1b[2J') + self.joined_again += (carry + got).count(b'joined again') + self.feed(got) + if self.log: + self.log.write(got) + if b'\x1b[5n' in seen: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + with self.wlock: + try: + os.write(self.master, b'\x1b[0n') + except OSError: + return + + SGR = re.compile(rb'\x1b\[[0-9:;]*m') + + def feed(self, got): + """To pyte, without its colours (pyte reads no colon sub-parameters) + and holding back a sequence a read cut in two.""" + data = getattr(self, 'held_back', b'') + got + cut = data.rfind(b'\x1b') + self.held_back = b'' + if cut >= 0 and not re.match(rb'\x1b(\[[0-9:;?<>=! ]*[@-~]|[^\[\]]|\][^\x07\x1b]*(\x07|\x1b\\))', data[cut:]): + data, self.held_back = data[:cut], data[cut:] + try: + self.stream.feed(self.SGR.sub(b'', data)) + except Exception: + pass + + def write(self, data): + """Keys as typed: each Esc alone, 80 ms clear after it.""" + parts = re.split(b'(\x1b(?![\\[O\\]n0-9a-z]))', data) + for part in parts: + if not part: + continue + if part == ESC: + with self.wlock: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + os.write(self.master, ESC) + self.esc_until = now() + 0.08 + time.sleep(0.08) + else: + with self.wlock: + wait = self.esc_until - now() + if wait > 0: + time.sleep(wait) + os.write(self.master, part) + + def resize(self, rows, cols): + self.rows, self.cols = rows, cols + self.px = (cols * 16, rows * 33) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + with self.lock: + self.screen.resize(rows, cols) + + def view(self): + with self.lock: + return [line.rstrip() for line in self.screen.display], (self.screen.cursor.y, self.screen.cursor.x), self.screen.cursor.hidden + + def spawn(self, argv, env, cwd, err): + slave = self.slave + return subprocess.Popen(argv, env=env, cwd=cwd, stdin=slave, stdout=slave, stderr=err, start_new_session=True, + preexec_fn=lambda: fcntl.ioctl(0, termios.TIOCSCTTY, 0)) + + def close(self): + self.closing = True + self.reader.join(timeout=2) + for fd in (self.slave, self.master): + try: + os.close(fd) + except OSError: + pass + + +# ---- clients ---- + +class Base: + def __init__(self, run, kind, label, link): + self.run = run + self.kind = kind + self.label = label + self.link = link # where it dials: a socket path, or the box + self.dir = run.root / label + self.dir.mkdir() + self.check = self.dir / 'check' + self.stats = self.dir / 'stats' + self.err = self.dir / 'err' + self.p = None + self.started = None + self.asked_to_end = False + self.gone = None # how it ended, if it did + self.inputs = 0 + self.lat = [] # ms, input to output/frame + self.no_output = 0 + self.held = threading.Event() # the monkey waits while set + self.paused = threading.Event() # set by the monkey once it waits + self.lock = threading.Lock() + + def env(self): + e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats)) + if self.run.args.trace: + e['PARDES_REPLICA_TRACE'] = '1' + if self.run.far: + e['PARDES_REPLICA_FAR'] = '1' + return e + + def attach_arg(self): + return f'--attach={self.link}:{NAME}' + + def links(self): + return [line for line in checks(self.stats) if line.startswith('inputs ')] + + def stats_sum(self): + keys = ('inputs', 'at_once', 'stopped', 'units', 'fast', 'neutral', 'mispredicted', 'foreign', 'visible') + tot = dict.fromkeys(keys, 0) + for line in self.links(): + w = line.split() + d = {w[i]: w[i + 1] for i in range(0, len(w) - 1, 2)} + for k in keys: + try: + tot[k] += int(d.get(k, 0)) + except ValueError: + pass + return tot + + def check_lines(self): + return checks(self.check) + + def poll(self): + if self.gone is None and self.p is not None and self.p.poll() is not None: + self.gone = self.p.returncode + return self.gone + + +class TtyClient(Base): + def __init__(self, run, label, link, rows=30, cols=100): + super().__init__(run, 'tty', label, link) + self.term = Term(rows, cols) + self.term.log = open(self.dir / 'out', 'wb', buffering=0) if run.args.keep else None + + def start(self): + argv = [self.run.pardes, self.attach_arg()] + # Its stderr a file: a panic's words and the replica's trace. + self.p = self.term.spawn(argv, self.env(), self.run.root, open(self.err, 'w')) + self.started = now() + + def ready(self): + return self.term.out_bytes > 2000 and now() - self.term.last_out > 0.3 + + def keys(self, data): + self.inputs += 1 + self.run.note(f'{self.label} < {data!r}') + before = self.term.out_bytes + t = now() + self.term.write(data) + threading.Thread(target=self._latency, args=(t, before), daemon=True).start() + + def _latency(self, t, before): + deadline = t + 2.0 + while now() < deadline: + with self.term.lock: + got = [c for c in self.term.chunks if c[0] >= t] + if got: + with self.lock: + self.lat.append((got[0][0] - t) * 1000) + return + time.sleep(0.005) + if not self.held.is_set(): + with self.lock: + self.no_output += 1 + + def mouse(self, kind, button, x, y): + # SGR: cells from 1; x, y here are fractions of the screen. + col = max(1, min(self.term.cols, int(x * self.term.cols) + 1)) + row = max(1, min(self.term.rows, int(y * self.term.rows) + 1)) + b = {'down': button - 1, 'up': button - 1, 'motion': 35, 'wheel': 64 if button > 0 else 65}[kind] + end = 'm' if kind == 'up' else 'M' + self.keys(f'\x1b[<{b};{col};{row}{end}'.encode()) + + def resize(self, rows, cols): + self.term.resize(rows, cols) + if self.p: + try: + os.killpg(self.p.pid, signal.SIGWINCH) + except OSError: + pass + + def said(self): + try: + return self.err.read_text(errors='replace') + except FileNotFoundError: + return '' + + def said_joined(self): + return self.term.joined_again + + def hangup(self): + """As its terminal closing: the normal way out (its stats written).""" + try: + os.killpg(self.p.pid, signal.SIGHUP) + except OSError: + pass + + def close(self): + self.term.close() + + +class WinClient(Base): + """A window client in test mode: offscreen, its keys and pointer through + its test feed on a pty (Escape as CSI 27 u: a lone ESC waits for more).""" + + def __init__(self, run, label, link, rows=34, cols=110): + super().__init__(run, 'win', label, link) + self.cap = self.dir / 'cap' + self.cap.mkdir() + self.latency = self.dir / 'latency' + self.master, self.slave = os.openpty() + ttymod.setraw(self.slave) + self.rows, self.cols = rows, cols + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + self.wlock = threading.Lock() + self.px = (cols * 16, rows * 33) # test mode's 27 px font, about + + def start(self): + e = dict(self.env(), PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(self.cap), PARDES_TEST_LATENCY=str(self.latency), + SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(self.cols), PARDES_TEST_ROWS=str(self.rows)) + self.p = subprocess.Popen([self.run.gui, self.attach_arg()], env=e, cwd=self.run.root, stdin=self.slave, + stdout=subprocess.DEVNULL, stderr=open(self.err, 'w'), start_new_session=True) + self.started = now() + + def ready(self): + # Software presenting (offscreen) captures no ppm: its first frame is + # its latency trace's first F. + try: + return b'F ' in self.latency.read_bytes() and self.p.poll() is None + except FileNotFoundError: + return False + + def keys(self, data): + self.inputs += 1 + self.run.note(f'{self.label} < {data!r}') + with self.wlock: + try: + os.write(self.master, data.replace(ESC + b'[', b'\x00CSI').replace(ESC, b'\x1b[27u').replace(b'\x00CSI', ESC + b'[')) + except OSError: + pass + + def mouse(self, kind, button, x, y): + w, h = self.px + px, py = int(x * w), int(y * h) + if kind == 'wheel': + seq = f'\x1b]777;mouse;wheel;{button};{px};{py}\x07' + else: + seq = f'\x1b]777;mouse;{kind};{button};{px};{py}\x07' + self.inputs += 1 + self.run.note(f'{self.label} < mouse {kind} {button} {px} {py}') + with self.wlock: + try: + os.write(self.master, seq.encode()) + except OSError: + pass + + def resize(self, rows, cols): + self.rows, self.cols = rows, cols + self.px = (cols * 16, rows * 33) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0)) + + def hangup(self): + """Its test feed's end: the normal way out (its stats written).""" + try: + os.close(self.master) + except OSError: + pass + + def said(self): + try: + return self.err.read_text(errors='replace') + except FileNotFoundError: + return '' + + def said_joined(self): + return self.said().count('joined again') + + def frame_size(self): + try: + with open(self.cap / 'latest.ppm', 'rb') as f: + head = f.read(32).split() + self.px = (int(head[1]), int(head[2])) + except (OSError, IndexError, ValueError): + pass + + def latencies(self): + """Each stdin read to the end of the first frame after it, ms; and the + reads no frame followed within 2 s (a frame came after a later one).""" + try: + lines = self.latency.read_text().split('\n') + except FileNotFoundError: + return [], 0 + reads, frames = [], [] + for line in lines: + w = line.split() + if len(w) >= 2 and w[0] == 'R': + reads.append(int(w[1])) + elif len(w) >= 3 and w[0] == 'F': + frames.append(int(w[2])) + out, missed, j = [], 0, 0 + for r in reads: + while j < len(frames) and frames[j] < r: + j += 1 + if j < len(frames): + d = (frames[j] - r) / 1e6 + if d <= 2000: + out.append(d) + else: + missed += 1 + return out, missed + + def close(self): + for fd in (self.slave, self.master): + try: + os.close(fd) + except OSError: + pass + + +# ---- what a person does ---- + +def words(rng, n): + return ' '.join(rng.choice(WORDS) for _ in range(n)).encode() + + +def burst(c, rng): + """One action of a person's, in whatever state the client is.""" + k = c.keys + acts = c.run.actions + choice = rng.choices(acts, weights=[a[1] for a in acts])[0] + choice[2](c, rng, k) + return choice[0] + + +def act_move(c, rng, k): + for _ in range(rng.randint(1, 6)): + k(rng.choice([b'j', b'k', b'h', b'l', b'w', b'b', b'e', b'5j', b'3k', b'gg', b'ge', b'\x04', b'\x15', + b'\x06', b'\x02', b'}', b'{', b'%', b'x', b'12G', b'gl', b'gh'])) + time.sleep(rng.uniform(0.02, 0.2)) + + +def act_search(c, rng, k): + k(b'/' + rng.choice(WORDS).encode() + b'\r') + for _ in range(rng.randint(0, 3)): + time.sleep(rng.uniform(0.1, 0.4)) + k(rng.choice([b'n', b'N'])) + + +def act_jump(c, rng, k): + k(rng.choice([b'\x0f', b' jo', b' jj', b'gd', b'\x0f\x0f'])) + + +def act_insert(c, rng, k): + k(rng.choice([b'i', b'a', b'o', b'O', b'A'])) + for w in words(rng, rng.randint(1, 5)).split(): + k(w + b' ') + time.sleep(rng.uniform(0.03, 0.15)) + k(ESC) + + +def act_delete(c, rng, k): + k(rng.choice([b'xd', b'd', b'wd', b'xxd', b'c' + words(rng, 1) + ESC])) + + +def act_undo(c, rng, k): + k(rng.choice([b'u', b'uu', b'U', b'u'])) + + +def act_save(c, rng, k): + k(b' fs') + + +def act_look(c, rng, k): + # Enter is Look: a directory's row opens it, a word finds its next place. + k(rng.choice([b'\r', b'j\r', b'jj\r', b'gg\r'])) + + +def act_panes(c, rng, k): + k(b' bb') + time.sleep(rng.uniform(0.3, 1.0)) + k(rng.choice([b'\r', ESC, b'jj\r', b'not\r'])) + + +def act_find(c, rng, k): + k(rng.choice([b' ff', b' fr', b' fg'])) + time.sleep(rng.uniform(0.3, 1.0)) + k(rng.choice([b'notes\r', ESC, b'j\r', b'inner\r'])) + + +def act_shell(c, rng, k): + # A terminal of its own, a program in it, and back out to normal mode. + k(b' nt') + time.sleep(rng.uniform(0.8, 1.5)) + for _ in range(rng.randint(1, 3)): + prog = rng.choice(['ls -la\r', 'seq 1 3000\r', 'less big.txt\r', 'vi notes.txt\r', 'top\r', 'sleep 30\r', + 'echo ' + rng.choice(WORDS) + '\r', 'cat code.zig\r']) + k(prog.encode()) + time.sleep(rng.uniform(0.5, 2.0)) + if prog.startswith('less'): + # Never SPC then q: that is Exit, should the keys reach no less. + k(b'jjjj\x06') + time.sleep(0.3) + k(b'q') + elif prog.startswith('vi'): + k(b'jjjjx') + k(ESC) + k(b':q!\r') + elif prog.startswith(('top', 'sleep', 'seq')): + k(b'\x03' if not prog.startswith('top') else b'q') + time.sleep(rng.uniform(0.2, 0.8)) + k(b'\x02') # Ctrl-b: normal mode + if rng.random() < 0.5 and c.run.pane_count() >= 4: + time.sleep(0.3) + k(b' d') + time.sleep(0.6) + k(rng.choice([b'k', b'j', b''])) + + +def act_window(c, rng, k): + k(b'\x17' + rng.choice([b'h', b'j', b'k', b'l'])) + + +def act_column(c, rng, k): + few = c.run.pane_count() < 5 + k(rng.choice([b' cn', b' cn'] if few else [b' cn', b' cn', b' cd', b' cj'])) + + +def act_del(c, rng, k): + if c.run.pane_count() < 4: + return act_move(c, rng, k) + k(b' d') + time.sleep(0.6) + k(rng.choice([b'k', b'j', b'', b' d'])) + + +def act_new(c, rng, k): + k(b' fn') + time.sleep(0.4) + k(b'i' + words(rng, 4)) + k(ESC) + + +def act_esc(c, rng, k): + k(rng.choice([ESC, b'\x02', b':', b':'])) + time.sleep(0.3) + k(ESC) + + +def act_click(c, rng, k): + x, y = rng.random(), rng.uniform(0.05, 1.0) + b = rng.choice([1, 1, 1, 3]) + c.mouse('down', b, x, y) + time.sleep(rng.uniform(0.03, 0.12)) + if rng.random() < 0.3: # a drag + c.mouse('motion', b, min(1, x + rng.uniform(0, 0.2)), min(1, y + rng.uniform(0, 0.1))) + c.mouse('up', b, x, y) + + +def act_wheel(c, rng, k): + x, y = rng.random(), rng.uniform(0.1, 1.0) + d = rng.choice([1, -1]) + for _ in range(rng.randint(1, 8)): + c.mouse('wheel', d, x, y) + time.sleep(rng.uniform(0.01, 0.08)) + + +def act_hover(c, rng, k): + x, y = rng.random(), rng.uniform(0.1, 1.0) + for _ in range(rng.randint(2, 12)): + x = min(1, max(0, x + rng.uniform(-0.05, 0.05))) + y = min(1, max(0.05, y + rng.uniform(-0.03, 0.03))) + c.mouse('motion', 0, x, y) + time.sleep(rng.uniform(0.01, 0.06)) + time.sleep(rng.uniform(0.05, 0.6)) # resting: a Look preview + + +def act_paste(c, rng, k): + k(b'i\x1b[200~' + b'\n'.join(words(rng, 6) for _ in range(rng.randint(1, 4))) + b'\x1b[201~') + k(ESC) + + +def act_settings(c, rng, k): + k(rng.choice([b' tw', b' ?', b' hm', b' tt'])) + time.sleep(0.5) + k(ESC) + + +def act_idle(c, rng, k): + time.sleep(rng.choice([0.5, 1, 2, 4, 8])) + + +ACTIONS = [ + ('move', 30, act_move), ('search', 5, act_search), ('jump', 4, act_jump), ('insert', 12, act_insert), + ('delete', 4, act_delete), ('undo', 5, act_undo), ('save', 3, act_save), ('look', 6, act_look), + ('panes', 3, act_panes), ('find', 3, act_find), ('shell', 3, act_shell), ('window', 4, act_window), + ('column', 1, act_column), ('del', 2, act_del), ('new', 2, act_new), ('esc', 4, act_esc), + ('click', 8, act_click), ('wheel', 8, act_wheel), ('hover', 6, act_hover), ('paste', 2, act_paste), + ('settings', 1, act_settings), ('idle', 6, act_idle), +] + + +def monkey(c, rng, stop): + while not stop.is_set(): + if c.held.is_set(): + c.paused.set() + time.sleep(0.05) + continue + c.paused.clear() + if c.poll() is not None: + return + try: + what = burst(c, rng) + c.run.note(f'{c.label} {what}') + except OSError: + return + time.sleep(rng.uniform(0.05, 0.6)) + + +# ---- the run ---- + +class Run: + def __init__(self, args, cell): + self.args = args + self.cell = cell + self.root = Path(tempfile.mkdtemp(prefix=f'soak-{cell}-', dir=os.environ.get('TMPDIR') or '/tmp')) + self.pardes = str(Path(args.pardes).resolve()) + self.gui = str(Path(args.gui).resolve()) + self.far = True + self.pids = [] + self.clients = [] + self.relays = [] + self.events = [] + self.breaches = [] + self.notes = open(self.root / 'actions', 'w') + only = set(args.only.split(',')) if args.only else None + skip = set(args.skip.split(',')) if args.skip else set() + self.actions = [a for a in ACTIONS if (only is None or a[0] in only) and a[0] not in skip] + self.nlock = threading.Lock() + self.t0 = now() + env = {k: v for k, v in os.environ.items() if not k.startswith(('PARDES_', 'WAYLAND', 'DBUS_', 'DISPLAY', 'BROWSER')) + and 'PAGER' not in k and k not in ('winid', 'EDITOR', 'NINE_MOUNT')} + env.update(HOME=str(self.root), XDG_RUNTIME_DIR=str(self.root), XDG_CONFIG_HOME=str(self.root / 'config'), + XDG_CACHE_HOME=str(self.root / 'cache'), XDG_STATE_HOME=str(self.root / 'state'), + TERM='xterm-256color', SHELL=shutil.which('bash') or '/bin/bash', PARDES_NOTIME='1') + (self.root / 'config' / 'pardes').mkdir(parents=True) + (self.root / 'config' / 'pardes' / 'init').write_text('Shell bash\nMessageLinger 0\n') + self.env = env + self.work = self.root / 'work' + self.nine = None + self.nine_lock = threading.RLock() + self.session_pid = None + + def note(self, what): + with self.nlock: + self.notes.write(f'{now() - self.t0:9.3f} {what}\n') + self.notes.flush() + + def breach(self, what): + self.note('BREACH ' + what) + self.breaches.append(what) + + def spawn(self, argv, **kw): + p = subprocess.Popen(argv, **kw) + self.pids.append(p.pid) + return p + + def relay(self, rtt): + if not rtt: + return self.root / 'pardes-replica.sock' + at = self.root / f'relay{rtt}.sock' + if not at.exists(): + self.relays.append(self.spawn([sys.executable, '-B', str(HERE / 'delay_relay.py'), str(at), str(self.root / 'pardes-replica.sock'), + '--delay', str(rtt / 2), '--jitter', str(self.args.jitter)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) + wait_for('the relay', at.exists) + return at + + def pane_count(self): + """The session's panes (a script's view of /index), cached a second: + a Del that closed the last one would end the session.""" + t = now() + if t - getattr(self, 'counted_at', 0) > 1: + rows = self.nine_read('/index') or b'' + self.counted, self.counted_at = len(rows.splitlines()), t + return self.counted + + def nine_read(self, path): + with self.nine_lock: + return self._nine_read(path) + + def nine_write(self, path, data): + with self.nine_lock: + return self._nine_write(path, data) + + def _nine_read(self, path): + for _ in range(3): + try: + if self.nine is None: + self.nine = Client(self.nine_sock, timeout=10) + return self.nine.read(path) + except (OSError, EOFError): + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None + time.sleep(0.5) + return None + + def _nine_write(self, path, data): + for _ in range(2): + try: + if self.nine is None: + self.nine = Client(self.nine_sock, timeout=10) + return self.nine.write(path, data) + except EOFError: + self.nine = None + time.sleep(0.5) + except OSError as e: + return e + + # The session: here. + def start_session(self): + rng = random.Random(self.args.seed) + fixture(self.work, rng) + e = dict(self.env, PARDES_REPLICA_STATS=str(self.root / 'session.stats')) + if self.args.trace: + e['PARDES_REPLICA_TRACE'] = '1' + self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work)], env=e, cwd=self.work, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / 'session.err', 'w'), + start_new_session=True) + self.nine_sock = self.root / f'pardes-9p-{NAME}.sock' + wait_for('the session\'s 9P socket', self.nine_sock.exists, timeout=20) + wait_for('the replica socket', (self.root / 'pardes-replica.sock').exists, timeout=20) + status = self.nine_read('/status').decode() + self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + self.pids.append(self.session_pid) + # Two shells beside the directory, as the user's had. + for _ in range(2): + self.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(0.5) + + def session_alive(self): + return alive(self.session_pid) if self.session_pid else False + + def end_session(self): + """Exit (twice: unsaved text asks once), then --kill if it stays.""" + for _ in range(3): + self.nine_write('/ctl', b'Exit\n') + try: + wait_for('the session\'s end', lambda: not self.session_alive(), timeout=3) + return True + except TimeoutError: + pass + self.note('the session did not Exit: --kill') + self.kill_session() + return False + + def kill_session(self): + subprocess.run([self.pardes, f'--kill={NAME}'], env=self.env, cwd=self.root, capture_output=True, timeout=30) + + def session_said(self): + try: + return (self.root / 'session.err').read_text(errors='replace') + except FileNotFoundError: + return '' + + def crashes(self): + p = self.root / 'config' / 'pardes' / 'crashes' + return p.read_text() if p.exists() else '' + + def cleanup(self): + for c in self.clients: + if c.p and c.p.poll() is None: + c.p.kill() + c.close() + for pid in self.pids + processes(self.root): + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + + +class BoxRun(Run): + """The session on the FreeBSD box; the clients here through ssh.""" + + def __init__(self, args, cell): + super().__init__(args, cell) + self.far = False # an ssh client is far by itself + self.run_dir = f'{BOX_T}/run-hunt' + self.home = f'{BOX_T}/home-hunt' + self.env['PARDES_SSH'] = ' '.join(SSH) + self.env['PARDES_REMOTE_RUNTIME'] = self.run_dir + self.fwd = None + + def box(self, script, timeout=60): + return subprocess.run(SSH + [BOX, script], capture_output=True, timeout=timeout) + + def benv(self): + return (f'HOME={self.home} XDG_RUNTIME_DIR={self.run_dir} XDG_CONFIG_HOME={self.home}/config TMPDIR={BOX_T}/tmp-hunt ' + f'XDG_CACHE_HOME={self.home}/cache XDG_STATE_HOME={self.home}/state PATH={BOX_T}/bin:/usr/local/bin:/usr/bin:/bin ' + f'SHELL=/usr/local/bin/bash TERM=xterm-256color PARDES_NOTIME=1 PARDES_REPLICA_STATS={self.home}/session.stats' + + (' PARDES_REPLICA_TRACE=1' if self.args.trace else '')) + + def start_session(self): + rng = random.Random(self.args.seed) + fixture(self.work, rng) + tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout + bin_ = self.args.remote_bin + script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; ' + f'rm -rf {self.home}; mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; ' + f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; ' + f'tar -C {self.home}/work -xf -; cd {self.home}/work; ' + f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work </dev/null >{self.home}/session.err 2>&1 & ' + f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; ' + f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err; exit 4; }}') + got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120) + if got.returncode != 0: + raise RuntimeError(f'box session: {got.returncode} {got.stdout.decode()} {got.stderr.decode()}') + self.nine_sock = self.root / 'box9p.sock' + self.fwd = self.spawn(SSH + ['-N', '-o', 'StreamLocalBindUnlink=yes', '-L', f'{self.nine_sock}:{self.run_dir}/pardes-9p-{NAME}.sock', BOX], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + wait_for('the 9P forward', self.nine_sock.exists, timeout=20) + status = self.nine_read('/status').decode() + self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid']) + for _ in range(2): + self.nine_write('/pane/1/exec', b'Tty\n') + time.sleep(1) + + def session_alive(self): + got = self.box(f'kill -0 {self.session_pid} 2>/dev/null && echo yes || echo no', timeout=30) + return got.stdout.strip() == b'yes' + + def session_said(self): + got = self.box(f'cat {self.home}/session.err; ls {self.home}/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60) + return got.stdout.decode(errors='replace') + + def crashes(self): + return '' + + def kill_session(self): + self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60) + + def relay(self, rtt): + return BOX + + def cleanup(self): + super().cleanup() + # Everything started under run-hunt, by its environment: ps -e prints it. + self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; ' + f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60) + + +def start_client(run, kind, rtt, n): + label = f'{kind}{rtt}-{n}' + link = run.relay(rtt) + c = (TtyClient if kind == 'tty' else WinClient)(run, label, link) + c.rtt = rtt + c.start() + run.pids.append(c.p.pid) + run.clients.append(c) + run.note(f'{label} started (pid {c.p.pid})') + wait_for(f'{label}\'s first frame', c.ready, timeout=60 if isinstance(run, BoxRun) else 30) + if isinstance(c, WinClient): + c.frame_size() + time.sleep(0.3) + return c + + +def hold(run, clients, timeout=30): + for c in clients: + c.held.set() + deadline = now() + timeout + for c in clients: + while not c.paused.is_set() and c.poll() is None and now() < deadline: + time.sleep(0.05) + + +def release(clients): + for c in clients: + c.held.clear() + + +def quiet_point(run, live): + """Every monkey held: each client must give a fresh check within 8 s + (a freeze otherwise), and none may differ; a terminal client's view + against the session's.""" + hold(run, live) + time.sleep(1.5) + for c in live: + if c.poll() is not None: + continue + before = len(c.check_lines()) + # A key that moves nothing a digest sees but the view's cursor, so the + # client's input is the session's last: its view is the live one. + c.keys(b'l') + t = now() + fresh = False + while now() - t < 8 + 2 * c.rtt / 1000: + if len(c.check_lines()) > before: + fresh = True + break + time.sleep(0.1) + if not fresh and c.poll() is None: + run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point') + if isinstance(c, TtyClient) and fresh: + view_check(run, c) + release(live) + + +def view_check(run, c): + """The terminal client's screen and cursor against what the session renders + for the same view (its input was the last, so 9P reads that view).""" + time.sleep(0.4) + raw = run.nine_read('/screen') + if not raw: + return + try: + s = json.loads(raw) + except ValueError: + return + lines, cursor, hidden = c.term.view() + cols, rows = s.get('cols'), s.get('rows') + if (cols, rows) != (c.term.cols, c.term.rows): + c.view_skips = getattr(c, 'view_skips', 0) + 1 + return + theirs = [] + cells = s.get('cells', []) + for r in range(rows): + row = cells[r * cols:(r + 1) * cols] if len(cells) == rows * cols else (cells[r] if r < len(cells) else []) + theirs.append(''.join(cell[0] if isinstance(cell, list) else str(cell) for cell in row).rstrip()) + sc = s.get('cursor') + differ = [r for r in range(min(rows, len(lines))) if lines[r] != theirs[r]] + c.views = getattr(c, 'views', 0) + 1 + want = None + if isinstance(sc, dict): + want = (sc.get('row', sc.get('y')), sc.get('col', sc.get('x'))) + elif isinstance(sc, list) and len(sc) >= 2: + want = (sc[1], sc[0]) + bad_cursor = want is not None and None not in want and tuple(want) != cursor and not hidden + if differ or bad_cursor: + c.view_bad = getattr(c, 'view_bad', 0) + 1 + d = c.dir / f'view-{c.view_bad}.txt' + d.write_text(f'cursor client {cursor} hidden {hidden} session {sc}\nrows differing {differ[:12]}\n' + + ''.join(f'client {r:2} |{lines[r]}|\nsession {r:2} |{theirs[r]}|\n' for r in differ[:12])) + run.breach(f'{c.label}: its screen is not the session\'s render of its view ({len(differ)} rows' + f'{", cursor " + str(cursor) + " vs " + str(want) if bad_cursor else ""}): {d}') + + +def global_event(run, rng, live, made): + """Something happens beside the typing: a resize, a client detached and + another attached, an Upgrade, a script's edit over 9P.""" + kinds = [e for e in run.args.events.split(',') if e] + if not kinds: + return None + if run.args.upgrade and not getattr(run, 'upgraded', False): + kinds.append('upgrade') + what = rng.choice(kinds) + if what == 'resize': + c = rng.choice(live) + rows, cols = rng.randint(18, 50), rng.randint(60, 160) + run.note(f'{c.label} resized to {cols}x{rows}') + c.resize(rows, cols) + elif what == 'detach': + c = rng.choice(live) + run.note(f'{c.label} detached') + hold(run, [c]) + c.asked_to_end = True + c.keys(ESC) + c.keys(b'\x02') + time.sleep(0.2) + c.keys(ESC) + c.keys(b' sD') + try: + wait_for(f'{c.label} let go', lambda: c.poll() is not None, timeout=15) + except TimeoutError: + # Its keys went to a program or a prompt: it is ended as a window closes. + run.note(f'{c.label} did not detach by keys: hung up') + c.hangup() + wait_for(f'{c.label} hung up', lambda: c.poll() is not None, timeout=15) + live.remove(c) + made[0] += 1 + n = start_client(run, c.kind, c.rtt, made[0]) + live.append(n) + return n + elif what == 'upgrade': + run.note('Upgrade') + run.upgraded = True + run.nine_write('/ctl', b'Upgrade\n') + time.sleep(3) + if run.nine: + try: + run.nine.socket.close() + except Exception: + pass + run.nine = None + elif what == 'script': + rows = (run.nine_read('/index') or b'').splitlines() + texts = [r.split()[0].decode() for r in rows if len(r.split()) > 1 and r.split()[1] == b'text'] + if texts: + n = rng.choice(texts) + run.note(f'9P append to pane {n}') + run.nine_write(f'/pane/{n}/body', b'a line a script wrote\n') + return None + + +def soak(run, kinds, rtt, minutes, seed): + rng = random.Random(seed) + stop = threading.Event() + live = [] + threads = [] + made = [0] + try: + soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made) + except Exception: + import traceback + run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) + finally: + stop.set() + report_cell(run, rtt) + run.cleanup() + + +def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): + run.start_session() + for kind in kinds: + made[0] += 1 + live.append(start_client(run, kind, rtt, made[0])) + for c in list(live): + t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + end = now() + minutes * 60 + next_quiet = now() + 45 + next_event = now() + rng.uniform(20, 60) + while now() < end: + time.sleep(0.5) + for c in list(live): + if c.poll() is not None: + run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}') + live.remove(c) + if not run.session_alive(): + run.breach('the session died') + break + if not live: + break + if now() > next_event: + n = global_event(run, rng, live, made) + if n: + t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) + next_event = now() + rng.uniform(20, 70) + if now() > next_quiet: + quiet_point(run, live) + next_quiet = now() + rng.uniform(40, 80) + stop.set() + for t in threads: + t.join(timeout=20) + if run.session_alive() and live: + quiet_point(run, live) + # Each client's links written: a hangup ends a client the normal way. + for c in live: + c.asked_to_end = True + if c.p.poll() is None: + c.hangup() + for c in live: + try: + wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20) + except TimeoutError: + run.breach(f'{c.label}: would not end on a hangup') + c.p.kill() + if run.session_alive(): + run.end_session() + + +def tail(c): + if isinstance(c, WinClient): + return c.said()[-300:].replace('\n', ' | ') + lines, _, _ = c.term.view() + return ' | '.join(line for line in lines if line.strip())[-300:] + + +ROWS = [] + + +def cores(run): + """systemd-coredump's cores since the cell began, of the pids it started.""" + if isinstance(run, BoxRun) or not shutil.which('coredumpctl'): + return [] + since = time.time() - (now() - run.t0) - 5 + got = subprocess.run(['coredumpctl', 'list', '--no-legend', '--no-pager', f'--since=@{since:.0f}'], capture_output=True, text=True) + ours = set(run.pids) | {c.p.pid for c in run.clients if c.p} | ({run.session_pid} if run.session_pid else set()) + out = [] + for line in got.stdout.splitlines(): + w = line.split() + pid = next((int(x) for x in w if x.isdigit() and int(x) in ours), None) + if pid: + out.append(f'pid {pid} {" ".join(x for x in w if x.startswith("SIG"))} {w[-2] if len(w) > 1 else ""}') + return out + + +def report_cell(run, rtt): + for core in cores(run): + run.breach(f'core dumped: {core} (coredumpctl debug <pid>)') + said = run.session_said() + for word in ('panic', 'Segmentation', 'abort', 'reached unreachable', 'core dumped'): + if word in said: + run.breach(f'session said "{word}": {said[said.find(word) - 200:said.find(word) + 300]!r}') + break + if run.crashes(): + run.breach(f'crashes: {run.crashes()[-400:]}') + for c in run.clients: + st = c.stats_sum() + links = len(c.links()) + lines = c.check_lines() + differ = [line for line in lines if line.startswith('differ')] + said = c.said_joined() + if links > 1 or said: + run.breach(f'{c.label}: rejoined (links {links}, "joined again" {said})') + if differ: + run.breach(f'{c.label}: {len(differ)} checks differ') + if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): + run.breach(f'{c.label}: died of signal {-c.gone}') + if c.gone not in (None, 0) and not c.asked_to_end: + pass # said when found + elif c.gone not in (None, 0, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: + run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}') + joined = c.said() + for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'): + if word in joined: + run.breach(f'{c.label} said "{word}": {joined[joined.find(word) - 200:joined.find(word) + 300]!r}') + break + if isinstance(c, WinClient): + lat, missed = c.latencies() + else: + lat, missed = c.lat, c.no_output + dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60) + ROWS.append({ + 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, + 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], + 'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, + 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), + 'view_bad': getattr(c, 'view_bad', 0), 'exit': c.gone, + }) + st = (run.root / 'session.stats') + text = '' + if isinstance(run, BoxRun): + text = run.box(f'cat {run.home}/session.stats 2>/dev/null', timeout=30).stdout.decode() + elif st.exists(): + text = st.read_text() + for line in text.splitlines(): + if line.startswith('session inputs'): + w = line.split() + d = {w[i]: w[i + 1] for i in range(1, len(w) - 1, 2)} + worst = int(d.get('boundary_us', '0/0/0').split('/')[2]) + ROWS.append({'cell': run.cell, 'client': 'session', 'rtt': rtt, 'boundary_us': d.get('boundary_us'), + 'input_us': d.get('input_us'), 'cut': d.get('cut')}) + if worst > 250_000: + run.breach(f'session: a boundary took {worst / 1000:.0f} ms') + run.note('end of cell') + + +def table(rows): + cols = ['cell', 'client', 'min', 'inputs', 'links', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', + 'p50', 'p99', 'nolat', 'views', 'view_bad', 'exit'] + def fmt(v): + if v is None: + return '-' + if isinstance(v, float): + return f'{v:.1f}' + return str(v) + out = [] + widths = {c: len(c) for c in cols} + body = [] + for r in rows: + if r['client'] == 'session': + body.append(f"{r['cell']:<8} session boundary_us p50/p99/max {r['boundary_us']} input_us {r['input_us']} cut {r['cut']}") + continue + vals = [fmt(r.get(c)) for c in cols] + for c, v in zip(cols, vals): + widths[c] = max(widths[c], len(v)) + body.append(vals) + out.append(' '.join(c.ljust(widths[c]) for c in cols)) + for b in body: + out.append(b if isinstance(b, str) else ' '.join(v.ljust(widths[c]) for c, v in zip(cols, b))) + return '\n'.join(out) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('where', choices=['local', 'box']) + ap.add_argument('--minutes', type=float, default=None) + ap.add_argument('--rtts', default='0,30,80,207') + ap.add_argument('--jitter', type=float, default=10, help='ms of jitter a chunk, the relay\'s') + ap.add_argument('--seed', type=int, default=1) + ap.add_argument('--clients', default='tty,win', help='kinds, one client each: tty,win,win...') + ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes')) + ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui')) + ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt') + ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests') + ap.add_argument('--no-upgrade', dest='upgrade', action='store_false') + ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)') + ap.add_argument('--keep', action='store_true', help='keep the run\'s files (and the terminals\' output)') + ap.add_argument('--only', default='', help='the monkey\'s actions, by name: move,insert,...') + ap.add_argument('--skip', default='', help='actions left out, by name') + ap.add_argument('--events', default='resize,resize,detach,script,script', help='the global events drawn from') + args = ap.parse_args() + kinds = args.clients.split(',') + runs = [] + if args.where == 'local': + minutes = args.minutes if args.minutes is not None else 2 + for i, rtt in enumerate(int(x) for x in args.rtts.split(',')): + run = Run(args, f'local{rtt}') + run.far = not args.near + runs.append(run) + print(f'soak: {run.cell}: {kinds} for {minutes} min, seed {args.seed + i}, in {run.root}', flush=True) + soak(run, kinds, rtt, minutes, args.seed + i) + else: + minutes = args.minutes if args.minutes is not None else 20 + run = BoxRun(args, 'box') + runs.append(run) + print(f'soak: box: {kinds} for {minutes} min, seed {args.seed}, in {run.root}', flush=True) + soak(run, kinds, 207, minutes, args.seed) + print(table(ROWS)) + breaches = [(r.cell, b) for r in runs for b in r.breaches] + for cell, b in breaches: + print(f'BREACH {cell}: {b}') + for r in runs: + if args.keep or r.breaches: + print(f'kept: {r.root}') + else: + shutil.rmtree(r.root, ignore_errors=True) + print('soak:', 'FAIL' if breaches else 'pass') + sys.exit(1 if breaches else 0) + + +if __name__ == '__main__': + main() |
