summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--test/delay_relay.py28
-rw-r--r--test/soak.py1318
2 files changed, 1336 insertions, 10 deletions
diff --git a/test/delay_relay.py b/test/delay_relay.py
index 9d57095f..c600a3aa 100644
--- a/test/delay_relay.py
+++ b/test/delay_relay.py
@@ -4,9 +4,11 @@ it accepts on one socket, connects each client on to another, and holds
every chunk `delay` ms each way, timed from its arrival, so a stream's
throughput is not serialised behind the delay and its bytes stay in order.
An optional rate cap makes the link a bottleneck too: a chunk leaves once
-the ones before it have crossed at that rate, then waits out the delay.
+the ones before it have crossed at that rate, then waits out the delay. An
+optional jitter adds up to that many ms more to each chunk's delay, at
+random, its order kept (a chunk never overtakes the one before it).
- python3 -B test/delay_relay.py LISTEN TARGET --delay MS [--rate MB/s]
+ python3 -B test/delay_relay.py LISTEN TARGET --delay MS [--rate MB/s] [--jitter MS]
python3 -B test/delay_relay.py --self-check
An address is a path (a Unix socket) or host:port (TCP). The one-way
@@ -17,6 +19,7 @@ adds at each, and with a cap.
import argparse
import os
import queue
+import random
import socket
import statistics
import sys
@@ -58,13 +61,15 @@ def listen(address):
return s
-def pump(src, dst, delay, rate, done):
+def pump(src, dst, delay, rate, done, jitter=0.0):
"""src -> dst, each chunk due `delay` s after it arrived (and after the
- ones before it crossed at `rate` bytes/s), in order; its end a shutdown."""
+ ones before it crossed at `rate` bytes/s), plus up to `jitter` s at
+ random but never before the chunk ahead of it; its end a shutdown."""
due_q = queue.SimpleQueue()
def reader():
free = 0.0 # when the capped link has sent what it holds
+ last = 0.0 # when the chunk before is due: order is kept
while True:
try:
data = src.recv(CHUNK)
@@ -74,7 +79,8 @@ def pump(src, dst, delay, rate, done):
if rate:
free = max(free, now) + len(data) / rate
now = free
- due_q.put((now + delay, data))
+ last = max(last, now + delay + (random.uniform(0, jitter) if jitter else 0))
+ due_q.put((last, data))
if not data:
return
@@ -98,9 +104,10 @@ def pump(src, dst, delay, rate, done):
threading.Thread(target=writer, daemon=True).start()
-def serve(listener, target, delay_ms, rate_mbs=None):
+def serve(listener, target, delay_ms, rate_mbs=None, jitter_ms=0):
"""Relays every connection `listener` accepts until it is closed."""
delay = delay_ms / 1000
+ jitter = jitter_ms / 1000
rate = rate_mbs * 1e6 if rate_mbs else None
while True:
try:
@@ -124,8 +131,8 @@ def serve(listener, target, delay_ms, rate_mbs=None):
a.close()
b.close()
- pump(client, upstream, delay, rate, done)
- pump(upstream, client, delay, rate, done)
+ pump(client, upstream, delay, rate, done, jitter)
+ pump(upstream, client, delay, rate, done, jitter)
# ---- the self-check ----
@@ -252,12 +259,13 @@ def main():
ap.add_argument('target')
ap.add_argument('--delay', type=float, required=True, help='one-way delay, ms')
ap.add_argument('--rate', type=float, help='a bandwidth cap, MB/s')
+ ap.add_argument('--jitter', type=float, default=0, help='up to this many ms more a chunk, at random')
a = ap.parse_args()
listener = listen(a.listen)
print(f'relaying {a.listen} -> {a.target}, {a.delay} ms each way'
- + (f', {a.rate} MB/s' if a.rate else ''), flush=True)
+ + (f', {a.rate} MB/s' if a.rate else '') + (f', jitter {a.jitter} ms' if a.jitter else ''), flush=True)
try:
- serve(listener, a.target, a.delay, a.rate)
+ serve(listener, a.target, a.delay, a.rate, a.jitter)
except KeyboardInterrupt:
pass
diff --git a/test/soak.py b/test/soak.py
new file mode 100644
index 00000000..f75371ef
--- /dev/null
+++ b/test/soak.py
@@ -0,0 +1,1318 @@
+#!/usr/bin/env python3
+"""A soak of replica clients: what a person does in them, through each link,
+measured (docs/next-daemon.md, the replica and GUI replica notes).
+
+ python3 -B test/soak.py local [--minutes 2] [--rtts 0,30,80,207] [--jitter 10]
+ [--seed 1] [--clients tty,win] [--pardes P] [--gui G] [--trace]
+ python3 -B test/soak.py box [--minutes 20] [--clients win,tty] [--remote-bin B]
+ [--pardes P] [--gui G] [--seed 1] [--trace]
+
+`local`: a session on this machine (`pardes --detach=soak <a directory>`, the
+shape of the user's: a directory pane, then two shells), and at each RTT its
+clients through test/delay_relay.py (with jitter), far as through ssh
+(PARDES_REPLICA_FAR=1: the blobs by hash, the stream in zstd). One cell an
+RTT, `--minutes` each. `box`: the session on the FreeBSD box under
+/root/pardes-test (`--remote-bin`, by default bin/pardes-hunt there, run in
+run-hunt and home-hunt), the clients here through ssh, as the user's window
+was; its 9P through an `ssh -L` of the session's socket.
+
+The clients: terminals (`pardes --attach`, in a pty here whose screen pyte
+keeps, frame requests answered) and windows (`pardes-gui --attach`, SDL's
+offscreen driver, keys and the mouse through its test feed on a pty, so it
+can be resized). Each runs a seeded monkey of a person's actions: moves of
+every kind (keys, clicks, jumps, search), scrolls (keys, the wheel), edits,
+undo and save, Look at files and directories, the Panes picker, Find, new
+terminals with programs (ls, a long output, less, vi, top, Ctrl-C), panes and
+columns opened, moved and closed, the pointer resting and moving, pastes,
+idle gaps. Between them, at random: a client resized, a client detached and
+another attached, an Upgrade, a 9P script's edit. `--seed` makes a run
+again (the timing differs, the actions do not).
+
+Measured (a row a client, then the session's):
+ - rejoins: links past the first (PARDES_REPLICA_STATS writes one a link)
+ and `joined again` said by it; checks (PARDES_REPLICA_CHECK) that differ;
+ - exits nobody asked for, a signal or a non-zero code, the session's death,
+ a panic or `crashes` line;
+ - freezes: a quiet point where a client gives no fresh check within 8 s,
+ and a terminal client's key with no output within 2 s while it was not
+ held; the session's worst boundary (its stats at its end);
+ - the view: at each quiet point, a terminal client's cursor and screen
+ (pyte) against what the session renders for its view (/screen read right
+ after its own input made its view the live one), and the panes on it;
+ - blink: visible rollbacks (`visible`), mispredicted inputs, a terminal
+ client's full clears, a minute;
+ - keystroke to output (terminal) and to a frame (window, PARDES_TEST_LATENCY)
+ p50 / p99.
+
+Prints a table, then each breach, and exits 1 if there is any. Everything
+started is ended by pid (and found by its environment); the run's files are
+kept under $TMPDIR/soak-* (`--keep`) or removed.
+"""
+import argparse
+import fcntl
+import json
+import os
+import random
+import re
+import shutil
+import signal
+import socket
+import statistics
+import struct
+import subprocess
+import sys
+import tempfile
+import termios
+import threading
+import time
+import tty as ttymod
+from pathlib import Path
+
+import pyte
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+from ninep import Client # noqa: E402
+from daemon import processes, alive, wait_for, checks # noqa: E402
+
+SSH = ['/usr/bin/ssh', '-F', '/dev/null', '-o', 'BatchMode=yes', '-o', 'IdentitiesOnly=yes',
+ '-o', 'UserKnownHostsFile=/home/goblin/.ssh/known_hosts', '-i', '/home/goblin/.ssh/id_ed25519']
+BOX_T = '/root/pardes-test'
+NAME = 'soak'
+ESC = b'\x1b'
+
+
+def now():
+ return time.monotonic()
+
+
+def pct(xs, p):
+ if not xs:
+ return None
+ xs = sorted(xs)
+ return xs[min(len(xs) - 1, len(xs) * p // 100)]
+
+
+# ---- the fixture: a directory such as a person works in ----
+
+WORDS = ('alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa '
+ 'quebec romeo sierra tango uniform victor whiskey xray yankee zulu').split()
+
+
+def fixture(at, rng):
+ at.mkdir(parents=True, exist_ok=True)
+ (at / 'notes.txt').write_text(''.join(f'line {i} of the notes: {rng.choice(WORDS)} {rng.choice(WORDS)} see code.zig:{i % 90 + 1}\n'
+ for i in range(300)))
+ (at / 'code.zig').write_text(''.join(f'pub fn f{i}(x: u32) u32 {{\n return x * {i} + {rng.randrange(1000)}; // {rng.choice(WORDS)}\n}}\n\n'
+ for i in range(120)))
+ (at / 'big.txt').write_text(''.join(f'{i:06d} {" ".join(rng.choice(WORDS) for _ in range(9))}\n' for i in range(8000)))
+ (at / 'README').write_text('A directory to soak in.\nnotes.txt code.zig big.txt sub/\nsub/inner.txt:3\n')
+ sub = at / 'sub'
+ sub.mkdir(exist_ok=True)
+ (sub / 'inner.txt').write_text(''.join(f'inner {i} {rng.choice(WORDS)}\n' for i in range(60)))
+ (sub / 'more.md').write_text('# More\n\n' + ''.join(f'- item {i} {rng.choice(WORDS)}\n' for i in range(40)))
+
+
+# ---- terminals ----
+
+class Term:
+ """A terminal: a pty whose output is kept, timed and fed to pyte, its
+ frame requests (CSI 5 n) answered as a terminal emulator does. An Esc is
+ written alone and nothing follows it for 80 ms, an answer included: the
+ client's parser takes an Esc and what follows within 50 ms as Alt."""
+
+ def __init__(self, rows, cols):
+ self.master, self.slave = os.openpty()
+ self.rows, self.cols = rows, cols
+ fcntl.ioctl(self.slave, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0))
+ self.lock = threading.Lock()
+ self.wlock = threading.Lock()
+ self.screen = pyte.Screen(cols, rows)
+ self.stream = pyte.ByteStream(self.screen)
+ self.out_bytes = 0
+ self.last_out = 0.0
+ self.chunks = [] # (time, bytes) of the last minute or so, for latencies
+ self.clears = 0
+ self.joined_again = 0
+ self.tail = b''
+ self.esc_until = 0.0
+ self.closing = False
+ self.log = None
+ self.reader = threading.Thread(target=self.drain, daemon=True)
+ self.reader.start()
+
+ def drain(self):
+ import select
+ while not self.closing:
+ try:
+ if not select.select([self.master], [], [], 0.05)[0]:
+ continue
+ got = os.read(self.master, 65536)
+ except OSError:
+ return
+ if not got:
+ return
+ t = now()
+ carry, self.tail = self.tail, (self.tail + got)[-11:]
+ seen = carry[-3:] + got
+ with self.lock:
+ self.out_bytes += len(got)
+ self.last_out = t
+ self.chunks.append((t, len(got)))
+ if len(self.chunks) > 4096:
+ del self.chunks[:2048]
+ # The carry is shorter than either word: a match holds a new byte.
+ self.clears += seen.count(b'\x1b[2J')
+ self.joined_again += (carry + got).count(b'joined again')
+ self.feed(got)
+ if self.log:
+ self.log.write(got)
+ if b'\x1b[5n' in seen:
+ wait = self.esc_until - now()
+ if wait > 0:
+ time.sleep(wait)
+ with self.wlock:
+ try:
+ os.write(self.master, b'\x1b[0n')
+ except OSError:
+ return
+
+ SGR = re.compile(rb'\x1b\[[0-9:;]*m')
+
+ def feed(self, got):
+ """To pyte, without its colours (pyte reads no colon sub-parameters)
+ and holding back a sequence a read cut in two."""
+ data = getattr(self, 'held_back', b'') + got
+ cut = data.rfind(b'\x1b')
+ self.held_back = b''
+ if cut >= 0 and not re.match(rb'\x1b(\[[0-9:;?<>=! ]*[@-~]|[^\[\]]|\][^\x07\x1b]*(\x07|\x1b\\))', data[cut:]):
+ data, self.held_back = data[:cut], data[cut:]
+ try:
+ self.stream.feed(self.SGR.sub(b'', data))
+ except Exception:
+ pass
+
+ def write(self, data):
+ """Keys as typed: each Esc alone, 80 ms clear after it."""
+ parts = re.split(b'(\x1b(?![\\[O\\]n0-9a-z]))', data)
+ for part in parts:
+ if not part:
+ continue
+ if part == ESC:
+ with self.wlock:
+ wait = self.esc_until - now()
+ if wait > 0:
+ time.sleep(wait)
+ os.write(self.master, ESC)
+ self.esc_until = now() + 0.08
+ time.sleep(0.08)
+ else:
+ with self.wlock:
+ wait = self.esc_until - now()
+ if wait > 0:
+ time.sleep(wait)
+ os.write(self.master, part)
+
+ def resize(self, rows, cols):
+ self.rows, self.cols = rows, cols
+ self.px = (cols * 16, rows * 33)
+ fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0))
+ with self.lock:
+ self.screen.resize(rows, cols)
+
+ def view(self):
+ with self.lock:
+ return [line.rstrip() for line in self.screen.display], (self.screen.cursor.y, self.screen.cursor.x), self.screen.cursor.hidden
+
+ def spawn(self, argv, env, cwd, err):
+ slave = self.slave
+ return subprocess.Popen(argv, env=env, cwd=cwd, stdin=slave, stdout=slave, stderr=err, start_new_session=True,
+ preexec_fn=lambda: fcntl.ioctl(0, termios.TIOCSCTTY, 0))
+
+ def close(self):
+ self.closing = True
+ self.reader.join(timeout=2)
+ for fd in (self.slave, self.master):
+ try:
+ os.close(fd)
+ except OSError:
+ pass
+
+
+# ---- clients ----
+
+class Base:
+ def __init__(self, run, kind, label, link):
+ self.run = run
+ self.kind = kind
+ self.label = label
+ self.link = link # where it dials: a socket path, or the box
+ self.dir = run.root / label
+ self.dir.mkdir()
+ self.check = self.dir / 'check'
+ self.stats = self.dir / 'stats'
+ self.err = self.dir / 'err'
+ self.p = None
+ self.started = None
+ self.asked_to_end = False
+ self.gone = None # how it ended, if it did
+ self.inputs = 0
+ self.lat = [] # ms, input to output/frame
+ self.no_output = 0
+ self.held = threading.Event() # the monkey waits while set
+ self.paused = threading.Event() # set by the monkey once it waits
+ self.lock = threading.Lock()
+
+ def env(self):
+ e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats))
+ if self.run.args.trace:
+ e['PARDES_REPLICA_TRACE'] = '1'
+ if self.run.far:
+ e['PARDES_REPLICA_FAR'] = '1'
+ return e
+
+ def attach_arg(self):
+ return f'--attach={self.link}:{NAME}'
+
+ def links(self):
+ return [line for line in checks(self.stats) if line.startswith('inputs ')]
+
+ def stats_sum(self):
+ keys = ('inputs', 'at_once', 'stopped', 'units', 'fast', 'neutral', 'mispredicted', 'foreign', 'visible')
+ tot = dict.fromkeys(keys, 0)
+ for line in self.links():
+ w = line.split()
+ d = {w[i]: w[i + 1] for i in range(0, len(w) - 1, 2)}
+ for k in keys:
+ try:
+ tot[k] += int(d.get(k, 0))
+ except ValueError:
+ pass
+ return tot
+
+ def check_lines(self):
+ return checks(self.check)
+
+ def poll(self):
+ if self.gone is None and self.p is not None and self.p.poll() is not None:
+ self.gone = self.p.returncode
+ return self.gone
+
+
+class TtyClient(Base):
+ def __init__(self, run, label, link, rows=30, cols=100):
+ super().__init__(run, 'tty', label, link)
+ self.term = Term(rows, cols)
+ self.term.log = open(self.dir / 'out', 'wb', buffering=0) if run.args.keep else None
+
+ def start(self):
+ argv = [self.run.pardes, self.attach_arg()]
+ # Its stderr a file: a panic's words and the replica's trace.
+ self.p = self.term.spawn(argv, self.env(), self.run.root, open(self.err, 'w'))
+ self.started = now()
+
+ def ready(self):
+ return self.term.out_bytes > 2000 and now() - self.term.last_out > 0.3
+
+ def keys(self, data):
+ self.inputs += 1
+ self.run.note(f'{self.label} < {data!r}')
+ before = self.term.out_bytes
+ t = now()
+ self.term.write(data)
+ threading.Thread(target=self._latency, args=(t, before), daemon=True).start()
+
+ def _latency(self, t, before):
+ deadline = t + 2.0
+ while now() < deadline:
+ with self.term.lock:
+ got = [c for c in self.term.chunks if c[0] >= t]
+ if got:
+ with self.lock:
+ self.lat.append((got[0][0] - t) * 1000)
+ return
+ time.sleep(0.005)
+ if not self.held.is_set():
+ with self.lock:
+ self.no_output += 1
+
+ def mouse(self, kind, button, x, y):
+ # SGR: cells from 1; x, y here are fractions of the screen.
+ col = max(1, min(self.term.cols, int(x * self.term.cols) + 1))
+ row = max(1, min(self.term.rows, int(y * self.term.rows) + 1))
+ b = {'down': button - 1, 'up': button - 1, 'motion': 35, 'wheel': 64 if button > 0 else 65}[kind]
+ end = 'm' if kind == 'up' else 'M'
+ self.keys(f'\x1b[<{b};{col};{row}{end}'.encode())
+
+ def resize(self, rows, cols):
+ self.term.resize(rows, cols)
+ if self.p:
+ try:
+ os.killpg(self.p.pid, signal.SIGWINCH)
+ except OSError:
+ pass
+
+ def said(self):
+ try:
+ return self.err.read_text(errors='replace')
+ except FileNotFoundError:
+ return ''
+
+ def said_joined(self):
+ return self.term.joined_again
+
+ def hangup(self):
+ """As its terminal closing: the normal way out (its stats written)."""
+ try:
+ os.killpg(self.p.pid, signal.SIGHUP)
+ except OSError:
+ pass
+
+ def close(self):
+ self.term.close()
+
+
+class WinClient(Base):
+ """A window client in test mode: offscreen, its keys and pointer through
+ its test feed on a pty (Escape as CSI 27 u: a lone ESC waits for more)."""
+
+ def __init__(self, run, label, link, rows=34, cols=110):
+ super().__init__(run, 'win', label, link)
+ self.cap = self.dir / 'cap'
+ self.cap.mkdir()
+ self.latency = self.dir / 'latency'
+ self.master, self.slave = os.openpty()
+ ttymod.setraw(self.slave)
+ self.rows, self.cols = rows, cols
+ fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0))
+ self.wlock = threading.Lock()
+ self.px = (cols * 16, rows * 33) # test mode's 27 px font, about
+
+ def start(self):
+ e = dict(self.env(), PARDES_TEST='1', PARDES_TEST_CAPTURE_DIR=str(self.cap), PARDES_TEST_LATENCY=str(self.latency),
+ SDL_VIDEO_DRIVER='offscreen', PARDES_TEST_COLS=str(self.cols), PARDES_TEST_ROWS=str(self.rows))
+ self.p = subprocess.Popen([self.run.gui, self.attach_arg()], env=e, cwd=self.run.root, stdin=self.slave,
+ stdout=subprocess.DEVNULL, stderr=open(self.err, 'w'), start_new_session=True)
+ self.started = now()
+
+ def ready(self):
+ # Software presenting (offscreen) captures no ppm: its first frame is
+ # its latency trace's first F.
+ try:
+ return b'F ' in self.latency.read_bytes() and self.p.poll() is None
+ except FileNotFoundError:
+ return False
+
+ def keys(self, data):
+ self.inputs += 1
+ self.run.note(f'{self.label} < {data!r}')
+ with self.wlock:
+ try:
+ os.write(self.master, data.replace(ESC + b'[', b'\x00CSI').replace(ESC, b'\x1b[27u').replace(b'\x00CSI', ESC + b'['))
+ except OSError:
+ pass
+
+ def mouse(self, kind, button, x, y):
+ w, h = self.px
+ px, py = int(x * w), int(y * h)
+ if kind == 'wheel':
+ seq = f'\x1b]777;mouse;wheel;{button};{px};{py}\x07'
+ else:
+ seq = f'\x1b]777;mouse;{kind};{button};{px};{py}\x07'
+ self.inputs += 1
+ self.run.note(f'{self.label} < mouse {kind} {button} {px} {py}')
+ with self.wlock:
+ try:
+ os.write(self.master, seq.encode())
+ except OSError:
+ pass
+
+ def resize(self, rows, cols):
+ self.rows, self.cols = rows, cols
+ self.px = (cols * 16, rows * 33)
+ fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack('HHHH', rows, cols, 0, 0))
+
+ def hangup(self):
+ """Its test feed's end: the normal way out (its stats written)."""
+ try:
+ os.close(self.master)
+ except OSError:
+ pass
+
+ def said(self):
+ try:
+ return self.err.read_text(errors='replace')
+ except FileNotFoundError:
+ return ''
+
+ def said_joined(self):
+ return self.said().count('joined again')
+
+ def frame_size(self):
+ try:
+ with open(self.cap / 'latest.ppm', 'rb') as f:
+ head = f.read(32).split()
+ self.px = (int(head[1]), int(head[2]))
+ except (OSError, IndexError, ValueError):
+ pass
+
+ def latencies(self):
+ """Each stdin read to the end of the first frame after it, ms; and the
+ reads no frame followed within 2 s (a frame came after a later one)."""
+ try:
+ lines = self.latency.read_text().split('\n')
+ except FileNotFoundError:
+ return [], 0
+ reads, frames = [], []
+ for line in lines:
+ w = line.split()
+ if len(w) >= 2 and w[0] == 'R':
+ reads.append(int(w[1]))
+ elif len(w) >= 3 and w[0] == 'F':
+ frames.append(int(w[2]))
+ out, missed, j = [], 0, 0
+ for r in reads:
+ while j < len(frames) and frames[j] < r:
+ j += 1
+ if j < len(frames):
+ d = (frames[j] - r) / 1e6
+ if d <= 2000:
+ out.append(d)
+ else:
+ missed += 1
+ return out, missed
+
+ def close(self):
+ for fd in (self.slave, self.master):
+ try:
+ os.close(fd)
+ except OSError:
+ pass
+
+
+# ---- what a person does ----
+
+def words(rng, n):
+ return ' '.join(rng.choice(WORDS) for _ in range(n)).encode()
+
+
+def burst(c, rng):
+ """One action of a person's, in whatever state the client is."""
+ k = c.keys
+ acts = c.run.actions
+ choice = rng.choices(acts, weights=[a[1] for a in acts])[0]
+ choice[2](c, rng, k)
+ return choice[0]
+
+
+def act_move(c, rng, k):
+ for _ in range(rng.randint(1, 6)):
+ k(rng.choice([b'j', b'k', b'h', b'l', b'w', b'b', b'e', b'5j', b'3k', b'gg', b'ge', b'\x04', b'\x15',
+ b'\x06', b'\x02', b'}', b'{', b'%', b'x', b'12G', b'gl', b'gh']))
+ time.sleep(rng.uniform(0.02, 0.2))
+
+
+def act_search(c, rng, k):
+ k(b'/' + rng.choice(WORDS).encode() + b'\r')
+ for _ in range(rng.randint(0, 3)):
+ time.sleep(rng.uniform(0.1, 0.4))
+ k(rng.choice([b'n', b'N']))
+
+
+def act_jump(c, rng, k):
+ k(rng.choice([b'\x0f', b' jo', b' jj', b'gd', b'\x0f\x0f']))
+
+
+def act_insert(c, rng, k):
+ k(rng.choice([b'i', b'a', b'o', b'O', b'A']))
+ for w in words(rng, rng.randint(1, 5)).split():
+ k(w + b' ')
+ time.sleep(rng.uniform(0.03, 0.15))
+ k(ESC)
+
+
+def act_delete(c, rng, k):
+ k(rng.choice([b'xd', b'd', b'wd', b'xxd', b'c' + words(rng, 1) + ESC]))
+
+
+def act_undo(c, rng, k):
+ k(rng.choice([b'u', b'uu', b'U', b'u']))
+
+
+def act_save(c, rng, k):
+ k(b' fs')
+
+
+def act_look(c, rng, k):
+ # Enter is Look: a directory's row opens it, a word finds its next place.
+ k(rng.choice([b'\r', b'j\r', b'jj\r', b'gg\r']))
+
+
+def act_panes(c, rng, k):
+ k(b' bb')
+ time.sleep(rng.uniform(0.3, 1.0))
+ k(rng.choice([b'\r', ESC, b'jj\r', b'not\r']))
+
+
+def act_find(c, rng, k):
+ k(rng.choice([b' ff', b' fr', b' fg']))
+ time.sleep(rng.uniform(0.3, 1.0))
+ k(rng.choice([b'notes\r', ESC, b'j\r', b'inner\r']))
+
+
+def act_shell(c, rng, k):
+ # A terminal of its own, a program in it, and back out to normal mode.
+ k(b' nt')
+ time.sleep(rng.uniform(0.8, 1.5))
+ for _ in range(rng.randint(1, 3)):
+ prog = rng.choice(['ls -la\r', 'seq 1 3000\r', 'less big.txt\r', 'vi notes.txt\r', 'top\r', 'sleep 30\r',
+ 'echo ' + rng.choice(WORDS) + '\r', 'cat code.zig\r'])
+ k(prog.encode())
+ time.sleep(rng.uniform(0.5, 2.0))
+ if prog.startswith('less'):
+ # Never SPC then q: that is Exit, should the keys reach no less.
+ k(b'jjjj\x06')
+ time.sleep(0.3)
+ k(b'q')
+ elif prog.startswith('vi'):
+ k(b'jjjjx')
+ k(ESC)
+ k(b':q!\r')
+ elif prog.startswith(('top', 'sleep', 'seq')):
+ k(b'\x03' if not prog.startswith('top') else b'q')
+ time.sleep(rng.uniform(0.2, 0.8))
+ k(b'\x02') # Ctrl-b: normal mode
+ if rng.random() < 0.5 and c.run.pane_count() >= 4:
+ time.sleep(0.3)
+ k(b' d')
+ time.sleep(0.6)
+ k(rng.choice([b'k', b'j', b'']))
+
+
+def act_window(c, rng, k):
+ k(b'\x17' + rng.choice([b'h', b'j', b'k', b'l']))
+
+
+def act_column(c, rng, k):
+ few = c.run.pane_count() < 5
+ k(rng.choice([b' cn', b' cn'] if few else [b' cn', b' cn', b' cd', b' cj']))
+
+
+def act_del(c, rng, k):
+ if c.run.pane_count() < 4:
+ return act_move(c, rng, k)
+ k(b' d')
+ time.sleep(0.6)
+ k(rng.choice([b'k', b'j', b'', b' d']))
+
+
+def act_new(c, rng, k):
+ k(b' fn')
+ time.sleep(0.4)
+ k(b'i' + words(rng, 4))
+ k(ESC)
+
+
+def act_esc(c, rng, k):
+ k(rng.choice([ESC, b'\x02', b':', b':']))
+ time.sleep(0.3)
+ k(ESC)
+
+
+def act_click(c, rng, k):
+ x, y = rng.random(), rng.uniform(0.05, 1.0)
+ b = rng.choice([1, 1, 1, 3])
+ c.mouse('down', b, x, y)
+ time.sleep(rng.uniform(0.03, 0.12))
+ if rng.random() < 0.3: # a drag
+ c.mouse('motion', b, min(1, x + rng.uniform(0, 0.2)), min(1, y + rng.uniform(0, 0.1)))
+ c.mouse('up', b, x, y)
+
+
+def act_wheel(c, rng, k):
+ x, y = rng.random(), rng.uniform(0.1, 1.0)
+ d = rng.choice([1, -1])
+ for _ in range(rng.randint(1, 8)):
+ c.mouse('wheel', d, x, y)
+ time.sleep(rng.uniform(0.01, 0.08))
+
+
+def act_hover(c, rng, k):
+ x, y = rng.random(), rng.uniform(0.1, 1.0)
+ for _ in range(rng.randint(2, 12)):
+ x = min(1, max(0, x + rng.uniform(-0.05, 0.05)))
+ y = min(1, max(0.05, y + rng.uniform(-0.03, 0.03)))
+ c.mouse('motion', 0, x, y)
+ time.sleep(rng.uniform(0.01, 0.06))
+ time.sleep(rng.uniform(0.05, 0.6)) # resting: a Look preview
+
+
+def act_paste(c, rng, k):
+ k(b'i\x1b[200~' + b'\n'.join(words(rng, 6) for _ in range(rng.randint(1, 4))) + b'\x1b[201~')
+ k(ESC)
+
+
+def act_settings(c, rng, k):
+ k(rng.choice([b' tw', b' ?', b' hm', b' tt']))
+ time.sleep(0.5)
+ k(ESC)
+
+
+def act_idle(c, rng, k):
+ time.sleep(rng.choice([0.5, 1, 2, 4, 8]))
+
+
+ACTIONS = [
+ ('move', 30, act_move), ('search', 5, act_search), ('jump', 4, act_jump), ('insert', 12, act_insert),
+ ('delete', 4, act_delete), ('undo', 5, act_undo), ('save', 3, act_save), ('look', 6, act_look),
+ ('panes', 3, act_panes), ('find', 3, act_find), ('shell', 3, act_shell), ('window', 4, act_window),
+ ('column', 1, act_column), ('del', 2, act_del), ('new', 2, act_new), ('esc', 4, act_esc),
+ ('click', 8, act_click), ('wheel', 8, act_wheel), ('hover', 6, act_hover), ('paste', 2, act_paste),
+ ('settings', 1, act_settings), ('idle', 6, act_idle),
+]
+
+
+def monkey(c, rng, stop):
+ while not stop.is_set():
+ if c.held.is_set():
+ c.paused.set()
+ time.sleep(0.05)
+ continue
+ c.paused.clear()
+ if c.poll() is not None:
+ return
+ try:
+ what = burst(c, rng)
+ c.run.note(f'{c.label} {what}')
+ except OSError:
+ return
+ time.sleep(rng.uniform(0.05, 0.6))
+
+
+# ---- the run ----
+
+class Run:
+ def __init__(self, args, cell):
+ self.args = args
+ self.cell = cell
+ self.root = Path(tempfile.mkdtemp(prefix=f'soak-{cell}-', dir=os.environ.get('TMPDIR') or '/tmp'))
+ self.pardes = str(Path(args.pardes).resolve())
+ self.gui = str(Path(args.gui).resolve())
+ self.far = True
+ self.pids = []
+ self.clients = []
+ self.relays = []
+ self.events = []
+ self.breaches = []
+ self.notes = open(self.root / 'actions', 'w')
+ only = set(args.only.split(',')) if args.only else None
+ skip = set(args.skip.split(',')) if args.skip else set()
+ self.actions = [a for a in ACTIONS if (only is None or a[0] in only) and a[0] not in skip]
+ self.nlock = threading.Lock()
+ self.t0 = now()
+ env = {k: v for k, v in os.environ.items() if not k.startswith(('PARDES_', 'WAYLAND', 'DBUS_', 'DISPLAY', 'BROWSER'))
+ and 'PAGER' not in k and k not in ('winid', 'EDITOR', 'NINE_MOUNT')}
+ env.update(HOME=str(self.root), XDG_RUNTIME_DIR=str(self.root), XDG_CONFIG_HOME=str(self.root / 'config'),
+ XDG_CACHE_HOME=str(self.root / 'cache'), XDG_STATE_HOME=str(self.root / 'state'),
+ TERM='xterm-256color', SHELL=shutil.which('bash') or '/bin/bash', PARDES_NOTIME='1')
+ (self.root / 'config' / 'pardes').mkdir(parents=True)
+ (self.root / 'config' / 'pardes' / 'init').write_text('Shell bash\nMessageLinger 0\n')
+ self.env = env
+ self.work = self.root / 'work'
+ self.nine = None
+ self.nine_lock = threading.RLock()
+ self.session_pid = None
+
+ def note(self, what):
+ with self.nlock:
+ self.notes.write(f'{now() - self.t0:9.3f} {what}\n')
+ self.notes.flush()
+
+ def breach(self, what):
+ self.note('BREACH ' + what)
+ self.breaches.append(what)
+
+ def spawn(self, argv, **kw):
+ p = subprocess.Popen(argv, **kw)
+ self.pids.append(p.pid)
+ return p
+
+ def relay(self, rtt):
+ if not rtt:
+ return self.root / 'pardes-replica.sock'
+ at = self.root / f'relay{rtt}.sock'
+ if not at.exists():
+ self.relays.append(self.spawn([sys.executable, '-B', str(HERE / 'delay_relay.py'), str(at), str(self.root / 'pardes-replica.sock'),
+ '--delay', str(rtt / 2), '--jitter', str(self.args.jitter)],
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL))
+ wait_for('the relay', at.exists)
+ return at
+
+ def pane_count(self):
+ """The session's panes (a script's view of /index), cached a second:
+ a Del that closed the last one would end the session."""
+ t = now()
+ if t - getattr(self, 'counted_at', 0) > 1:
+ rows = self.nine_read('/index') or b''
+ self.counted, self.counted_at = len(rows.splitlines()), t
+ return self.counted
+
+ def nine_read(self, path):
+ with self.nine_lock:
+ return self._nine_read(path)
+
+ def nine_write(self, path, data):
+ with self.nine_lock:
+ return self._nine_write(path, data)
+
+ def _nine_read(self, path):
+ for _ in range(3):
+ try:
+ if self.nine is None:
+ self.nine = Client(self.nine_sock, timeout=10)
+ return self.nine.read(path)
+ except (OSError, EOFError):
+ try:
+ self.nine.socket.close()
+ except Exception:
+ pass
+ self.nine = None
+ time.sleep(0.5)
+ return None
+
+ def _nine_write(self, path, data):
+ for _ in range(2):
+ try:
+ if self.nine is None:
+ self.nine = Client(self.nine_sock, timeout=10)
+ return self.nine.write(path, data)
+ except EOFError:
+ self.nine = None
+ time.sleep(0.5)
+ except OSError as e:
+ return e
+
+ # The session: here.
+ def start_session(self):
+ rng = random.Random(self.args.seed)
+ fixture(self.work, rng)
+ e = dict(self.env, PARDES_REPLICA_STATS=str(self.root / 'session.stats'))
+ if self.args.trace:
+ e['PARDES_REPLICA_TRACE'] = '1'
+ self.session = self.spawn([self.pardes, f'--detach={NAME}', str(self.work)], env=e, cwd=self.work,
+ stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=open(self.root / 'session.err', 'w'),
+ start_new_session=True)
+ self.nine_sock = self.root / f'pardes-9p-{NAME}.sock'
+ wait_for('the session\'s 9P socket', self.nine_sock.exists, timeout=20)
+ wait_for('the replica socket', (self.root / 'pardes-replica.sock').exists, timeout=20)
+ status = self.nine_read('/status').decode()
+ self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid'])
+ self.pids.append(self.session_pid)
+ # Two shells beside the directory, as the user's had.
+ for _ in range(2):
+ self.nine_write('/pane/1/exec', b'Tty\n')
+ time.sleep(0.5)
+
+ def session_alive(self):
+ return alive(self.session_pid) if self.session_pid else False
+
+ def end_session(self):
+ """Exit (twice: unsaved text asks once), then --kill if it stays."""
+ for _ in range(3):
+ self.nine_write('/ctl', b'Exit\n')
+ try:
+ wait_for('the session\'s end', lambda: not self.session_alive(), timeout=3)
+ return True
+ except TimeoutError:
+ pass
+ self.note('the session did not Exit: --kill')
+ self.kill_session()
+ return False
+
+ def kill_session(self):
+ subprocess.run([self.pardes, f'--kill={NAME}'], env=self.env, cwd=self.root, capture_output=True, timeout=30)
+
+ def session_said(self):
+ try:
+ return (self.root / 'session.err').read_text(errors='replace')
+ except FileNotFoundError:
+ return ''
+
+ def crashes(self):
+ p = self.root / 'config' / 'pardes' / 'crashes'
+ return p.read_text() if p.exists() else ''
+
+ def cleanup(self):
+ for c in self.clients:
+ if c.p and c.p.poll() is None:
+ c.p.kill()
+ c.close()
+ for pid in self.pids + processes(self.root):
+ try:
+ os.kill(pid, signal.SIGKILL)
+ except OSError:
+ pass
+ if self.nine:
+ try:
+ self.nine.socket.close()
+ except Exception:
+ pass
+
+
+class BoxRun(Run):
+ """The session on the FreeBSD box; the clients here through ssh."""
+
+ def __init__(self, args, cell):
+ super().__init__(args, cell)
+ self.far = False # an ssh client is far by itself
+ self.run_dir = f'{BOX_T}/run-hunt'
+ self.home = f'{BOX_T}/home-hunt'
+ self.env['PARDES_SSH'] = ' '.join(SSH)
+ self.env['PARDES_REMOTE_RUNTIME'] = self.run_dir
+ self.fwd = None
+
+ def box(self, script, timeout=60):
+ return subprocess.run(SSH + [BOX, script], capture_output=True, timeout=timeout)
+
+ def benv(self):
+ return (f'HOME={self.home} XDG_RUNTIME_DIR={self.run_dir} XDG_CONFIG_HOME={self.home}/config TMPDIR={BOX_T}/tmp-hunt '
+ f'XDG_CACHE_HOME={self.home}/cache XDG_STATE_HOME={self.home}/state PATH={BOX_T}/bin:/usr/local/bin:/usr/bin:/bin '
+ f'SHELL=/usr/local/bin/bash TERM=xterm-256color PARDES_NOTIME=1 PARDES_REPLICA_STATS={self.home}/session.stats'
+ + (' PARDES_REPLICA_TRACE=1' if self.args.trace else ''))
+
+ def start_session(self):
+ rng = random.Random(self.args.seed)
+ fixture(self.work, rng)
+ tar = subprocess.run(['tar', '-C', str(self.work), '-cf', '-', '.'], capture_output=True, check=True).stdout
+ bin_ = self.args.remote_bin
+ script = (f'set -e; [ -d {self.run_dir} ] && [ -n "$(ls {self.run_dir} 2>/dev/null)" ] && {{ echo "run-hunt busy"; ls {self.run_dir}; exit 3; }}; '
+ f'rm -rf {self.home}; mkdir -p {self.home}/work {self.home}/config/pardes {BOX_T}/tmp-hunt {self.run_dir}; chmod 700 {self.run_dir}; '
+ f'printf "Shell bash\\nMessageLinger 0\\n" > {self.home}/config/pardes/init; '
+ f'tar -C {self.home}/work -xf -; cd {self.home}/work; '
+ f'env -i {self.benv()} {bin_} --detach={NAME} {self.home}/work </dev/null >{self.home}/session.err 2>&1 & '
+ f'for i in $(seq 400); do [ -S {self.run_dir}/pardes-9p-{NAME}.sock ] && break; sleep 0.05; done; '
+ f'[ -S {self.run_dir}/pardes-9p-{NAME}.sock ] || {{ cat {self.home}/session.err; exit 4; }}')
+ got = subprocess.run(SSH + [BOX, script], input=tar, capture_output=True, timeout=120)
+ if got.returncode != 0:
+ raise RuntimeError(f'box session: {got.returncode} {got.stdout.decode()} {got.stderr.decode()}')
+ self.nine_sock = self.root / 'box9p.sock'
+ self.fwd = self.spawn(SSH + ['-N', '-o', 'StreamLocalBindUnlink=yes', '-L', f'{self.nine_sock}:{self.run_dir}/pardes-9p-{NAME}.sock', BOX],
+ stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+ wait_for('the 9P forward', self.nine_sock.exists, timeout=20)
+ status = self.nine_read('/status').decode()
+ self.session_pid = int(dict(line.split(maxsplit=1) for line in status.splitlines())['pid'])
+ for _ in range(2):
+ self.nine_write('/pane/1/exec', b'Tty\n')
+ time.sleep(1)
+
+ def session_alive(self):
+ got = self.box(f'kill -0 {self.session_pid} 2>/dev/null && echo yes || echo no', timeout=30)
+ return got.stdout.strip() == b'yes'
+
+ def session_said(self):
+ got = self.box(f'cat {self.home}/session.err; ls {self.home}/*.core {self.home}/work/*.core 2>/dev/null; cat {self.home}/config/pardes/crashes 2>/dev/null', timeout=60)
+ return got.stdout.decode(errors='replace')
+
+ def crashes(self):
+ return ''
+
+ def kill_session(self):
+ self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60)
+
+ def relay(self, rtt):
+ return BOX
+
+ def cleanup(self):
+ super().cleanup()
+ # Everything started under run-hunt, by its environment: ps -e prints it.
+ self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; '
+ f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60)
+
+
+def start_client(run, kind, rtt, n):
+ label = f'{kind}{rtt}-{n}'
+ link = run.relay(rtt)
+ c = (TtyClient if kind == 'tty' else WinClient)(run, label, link)
+ c.rtt = rtt
+ c.start()
+ run.pids.append(c.p.pid)
+ run.clients.append(c)
+ run.note(f'{label} started (pid {c.p.pid})')
+ wait_for(f'{label}\'s first frame', c.ready, timeout=60 if isinstance(run, BoxRun) else 30)
+ if isinstance(c, WinClient):
+ c.frame_size()
+ time.sleep(0.3)
+ return c
+
+
+def hold(run, clients, timeout=30):
+ for c in clients:
+ c.held.set()
+ deadline = now() + timeout
+ for c in clients:
+ while not c.paused.is_set() and c.poll() is None and now() < deadline:
+ time.sleep(0.05)
+
+
+def release(clients):
+ for c in clients:
+ c.held.clear()
+
+
+def quiet_point(run, live):
+ """Every monkey held: each client must give a fresh check within 8 s
+ (a freeze otherwise), and none may differ; a terminal client's view
+ against the session's."""
+ hold(run, live)
+ time.sleep(1.5)
+ for c in live:
+ if c.poll() is not None:
+ continue
+ before = len(c.check_lines())
+ # A key that moves nothing a digest sees but the view's cursor, so the
+ # client's input is the session's last: its view is the live one.
+ c.keys(b'l')
+ t = now()
+ fresh = False
+ while now() - t < 8 + 2 * c.rtt / 1000:
+ if len(c.check_lines()) > before:
+ fresh = True
+ break
+ time.sleep(0.1)
+ if not fresh and c.poll() is None:
+ run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point')
+ if isinstance(c, TtyClient) and fresh:
+ view_check(run, c)
+ release(live)
+
+
+def view_check(run, c):
+ """The terminal client's screen and cursor against what the session renders
+ for the same view (its input was the last, so 9P reads that view)."""
+ time.sleep(0.4)
+ raw = run.nine_read('/screen')
+ if not raw:
+ return
+ try:
+ s = json.loads(raw)
+ except ValueError:
+ return
+ lines, cursor, hidden = c.term.view()
+ cols, rows = s.get('cols'), s.get('rows')
+ if (cols, rows) != (c.term.cols, c.term.rows):
+ c.view_skips = getattr(c, 'view_skips', 0) + 1
+ return
+ theirs = []
+ cells = s.get('cells', [])
+ for r in range(rows):
+ row = cells[r * cols:(r + 1) * cols] if len(cells) == rows * cols else (cells[r] if r < len(cells) else [])
+ theirs.append(''.join(cell[0] if isinstance(cell, list) else str(cell) for cell in row).rstrip())
+ sc = s.get('cursor')
+ differ = [r for r in range(min(rows, len(lines))) if lines[r] != theirs[r]]
+ c.views = getattr(c, 'views', 0) + 1
+ want = None
+ if isinstance(sc, dict):
+ want = (sc.get('row', sc.get('y')), sc.get('col', sc.get('x')))
+ elif isinstance(sc, list) and len(sc) >= 2:
+ want = (sc[1], sc[0])
+ bad_cursor = want is not None and None not in want and tuple(want) != cursor and not hidden
+ if differ or bad_cursor:
+ c.view_bad = getattr(c, 'view_bad', 0) + 1
+ d = c.dir / f'view-{c.view_bad}.txt'
+ d.write_text(f'cursor client {cursor} hidden {hidden} session {sc}\nrows differing {differ[:12]}\n' +
+ ''.join(f'client {r:2} |{lines[r]}|\nsession {r:2} |{theirs[r]}|\n' for r in differ[:12]))
+ run.breach(f'{c.label}: its screen is not the session\'s render of its view ({len(differ)} rows'
+ f'{", cursor " + str(cursor) + " vs " + str(want) if bad_cursor else ""}): {d}')
+
+
+def global_event(run, rng, live, made):
+ """Something happens beside the typing: a resize, a client detached and
+ another attached, an Upgrade, a script's edit over 9P."""
+ kinds = [e for e in run.args.events.split(',') if e]
+ if not kinds:
+ return None
+ if run.args.upgrade and not getattr(run, 'upgraded', False):
+ kinds.append('upgrade')
+ what = rng.choice(kinds)
+ if what == 'resize':
+ c = rng.choice(live)
+ rows, cols = rng.randint(18, 50), rng.randint(60, 160)
+ run.note(f'{c.label} resized to {cols}x{rows}')
+ c.resize(rows, cols)
+ elif what == 'detach':
+ c = rng.choice(live)
+ run.note(f'{c.label} detached')
+ hold(run, [c])
+ c.asked_to_end = True
+ c.keys(ESC)
+ c.keys(b'\x02')
+ time.sleep(0.2)
+ c.keys(ESC)
+ c.keys(b' sD')
+ try:
+ wait_for(f'{c.label} let go', lambda: c.poll() is not None, timeout=15)
+ except TimeoutError:
+ # Its keys went to a program or a prompt: it is ended as a window closes.
+ run.note(f'{c.label} did not detach by keys: hung up')
+ c.hangup()
+ wait_for(f'{c.label} hung up', lambda: c.poll() is not None, timeout=15)
+ live.remove(c)
+ made[0] += 1
+ n = start_client(run, c.kind, c.rtt, made[0])
+ live.append(n)
+ return n
+ elif what == 'upgrade':
+ run.note('Upgrade')
+ run.upgraded = True
+ run.nine_write('/ctl', b'Upgrade\n')
+ time.sleep(3)
+ if run.nine:
+ try:
+ run.nine.socket.close()
+ except Exception:
+ pass
+ run.nine = None
+ elif what == 'script':
+ rows = (run.nine_read('/index') or b'').splitlines()
+ texts = [r.split()[0].decode() for r in rows if len(r.split()) > 1 and r.split()[1] == b'text']
+ if texts:
+ n = rng.choice(texts)
+ run.note(f'9P append to pane {n}')
+ run.nine_write(f'/pane/{n}/body', b'a line a script wrote\n')
+ return None
+
+
+def soak(run, kinds, rtt, minutes, seed):
+ rng = random.Random(seed)
+ stop = threading.Event()
+ live = []
+ threads = []
+ made = [0]
+ try:
+ soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made)
+ except Exception:
+ import traceback
+ run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:])
+ finally:
+ stop.set()
+ report_cell(run, rtt)
+ run.cleanup()
+
+
+def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made):
+ run.start_session()
+ for kind in kinds:
+ made[0] += 1
+ live.append(start_client(run, kind, rtt, made[0]))
+ for c in list(live):
+ t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True)
+ t.start()
+ threads.append(t)
+ end = now() + minutes * 60
+ next_quiet = now() + 45
+ next_event = now() + rng.uniform(20, 60)
+ while now() < end:
+ time.sleep(0.5)
+ for c in list(live):
+ if c.poll() is not None:
+ run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}')
+ live.remove(c)
+ if not run.session_alive():
+ run.breach('the session died')
+ break
+ if not live:
+ break
+ if now() > next_event:
+ n = global_event(run, rng, live, made)
+ if n:
+ t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True)
+ t.start()
+ threads.append(t)
+ next_event = now() + rng.uniform(20, 70)
+ if now() > next_quiet:
+ quiet_point(run, live)
+ next_quiet = now() + rng.uniform(40, 80)
+ stop.set()
+ for t in threads:
+ t.join(timeout=20)
+ if run.session_alive() and live:
+ quiet_point(run, live)
+ # Each client's links written: a hangup ends a client the normal way.
+ for c in live:
+ c.asked_to_end = True
+ if c.p.poll() is None:
+ c.hangup()
+ for c in live:
+ try:
+ wait_for(f'{c.label}\'s end', lambda: c.poll() is not None, timeout=20)
+ except TimeoutError:
+ run.breach(f'{c.label}: would not end on a hangup')
+ c.p.kill()
+ if run.session_alive():
+ run.end_session()
+
+
+def tail(c):
+ if isinstance(c, WinClient):
+ return c.said()[-300:].replace('\n', ' | ')
+ lines, _, _ = c.term.view()
+ return ' | '.join(line for line in lines if line.strip())[-300:]
+
+
+ROWS = []
+
+
+def cores(run):
+ """systemd-coredump's cores since the cell began, of the pids it started."""
+ if isinstance(run, BoxRun) or not shutil.which('coredumpctl'):
+ return []
+ since = time.time() - (now() - run.t0) - 5
+ got = subprocess.run(['coredumpctl', 'list', '--no-legend', '--no-pager', f'--since=@{since:.0f}'], capture_output=True, text=True)
+ ours = set(run.pids) | {c.p.pid for c in run.clients if c.p} | ({run.session_pid} if run.session_pid else set())
+ out = []
+ for line in got.stdout.splitlines():
+ w = line.split()
+ pid = next((int(x) for x in w if x.isdigit() and int(x) in ours), None)
+ if pid:
+ out.append(f'pid {pid} {" ".join(x for x in w if x.startswith("SIG"))} {w[-2] if len(w) > 1 else ""}')
+ return out
+
+
+def report_cell(run, rtt):
+ for core in cores(run):
+ run.breach(f'core dumped: {core} (coredumpctl debug <pid>)')
+ said = run.session_said()
+ for word in ('panic', 'Segmentation', 'abort', 'reached unreachable', 'core dumped'):
+ if word in said:
+ run.breach(f'session said "{word}": {said[said.find(word) - 200:said.find(word) + 300]!r}')
+ break
+ if run.crashes():
+ run.breach(f'crashes: {run.crashes()[-400:]}')
+ for c in run.clients:
+ st = c.stats_sum()
+ links = len(c.links())
+ lines = c.check_lines()
+ differ = [line for line in lines if line.startswith('differ')]
+ said = c.said_joined()
+ if links > 1 or said:
+ run.breach(f'{c.label}: rejoined (links {links}, "joined again" {said})')
+ if differ:
+ run.breach(f'{c.label}: {len(differ)} checks differ')
+ if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL):
+ run.breach(f'{c.label}: died of signal {-c.gone}')
+ if c.gone not in (None, 0) and not c.asked_to_end:
+ pass # said when found
+ elif c.gone not in (None, 0, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end:
+ run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}')
+ joined = c.said()
+ for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'):
+ if word in joined:
+ run.breach(f'{c.label} said "{word}": {joined[joined.find(word) - 200:joined.find(word) + 300]!r}')
+ break
+ if isinstance(c, WinClient):
+ lat, missed = c.latencies()
+ else:
+ lat, missed = c.lat, c.no_output
+ dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60)
+ ROWS.append({
+ 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links,
+ 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'],
+ 'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None,
+ 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0),
+ 'view_bad': getattr(c, 'view_bad', 0), 'exit': c.gone,
+ })
+ st = (run.root / 'session.stats')
+ text = ''
+ if isinstance(run, BoxRun):
+ text = run.box(f'cat {run.home}/session.stats 2>/dev/null', timeout=30).stdout.decode()
+ elif st.exists():
+ text = st.read_text()
+ for line in text.splitlines():
+ if line.startswith('session inputs'):
+ w = line.split()
+ d = {w[i]: w[i + 1] for i in range(1, len(w) - 1, 2)}
+ worst = int(d.get('boundary_us', '0/0/0').split('/')[2])
+ ROWS.append({'cell': run.cell, 'client': 'session', 'rtt': rtt, 'boundary_us': d.get('boundary_us'),
+ 'input_us': d.get('input_us'), 'cut': d.get('cut')})
+ if worst > 250_000:
+ run.breach(f'session: a boundary took {worst / 1000:.0f} ms')
+ run.note('end of cell')
+
+
+def table(rows):
+ cols = ['cell', 'client', 'min', 'inputs', 'links', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min',
+ 'p50', 'p99', 'nolat', 'views', 'view_bad', 'exit']
+ def fmt(v):
+ if v is None:
+ return '-'
+ if isinstance(v, float):
+ return f'{v:.1f}'
+ return str(v)
+ out = []
+ widths = {c: len(c) for c in cols}
+ body = []
+ for r in rows:
+ if r['client'] == 'session':
+ body.append(f"{r['cell']:<8} session boundary_us p50/p99/max {r['boundary_us']} input_us {r['input_us']} cut {r['cut']}")
+ continue
+ vals = [fmt(r.get(c)) for c in cols]
+ for c, v in zip(cols, vals):
+ widths[c] = max(widths[c], len(v))
+ body.append(vals)
+ out.append(' '.join(c.ljust(widths[c]) for c in cols))
+ for b in body:
+ out.append(b if isinstance(b, str) else ' '.join(v.ljust(widths[c]) for c, v in zip(cols, b)))
+ return '\n'.join(out)
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
+ ap.add_argument('where', choices=['local', 'box'])
+ ap.add_argument('--minutes', type=float, default=None)
+ ap.add_argument('--rtts', default='0,30,80,207')
+ ap.add_argument('--jitter', type=float, default=10, help='ms of jitter a chunk, the relay\'s')
+ ap.add_argument('--seed', type=int, default=1)
+ ap.add_argument('--clients', default='tty,win', help='kinds, one client each: tty,win,win...')
+ ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes'))
+ ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui'))
+ ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt')
+ ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests')
+ ap.add_argument('--no-upgrade', dest='upgrade', action='store_false')
+ ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)')
+ ap.add_argument('--keep', action='store_true', help='keep the run\'s files (and the terminals\' output)')
+ ap.add_argument('--only', default='', help='the monkey\'s actions, by name: move,insert,...')
+ ap.add_argument('--skip', default='', help='actions left out, by name')
+ ap.add_argument('--events', default='resize,resize,detach,script,script', help='the global events drawn from')
+ args = ap.parse_args()
+ kinds = args.clients.split(',')
+ runs = []
+ if args.where == 'local':
+ minutes = args.minutes if args.minutes is not None else 2
+ for i, rtt in enumerate(int(x) for x in args.rtts.split(',')):
+ run = Run(args, f'local{rtt}')
+ run.far = not args.near
+ runs.append(run)
+ print(f'soak: {run.cell}: {kinds} for {minutes} min, seed {args.seed + i}, in {run.root}', flush=True)
+ soak(run, kinds, rtt, minutes, args.seed + i)
+ else:
+ minutes = args.minutes if args.minutes is not None else 20
+ run = BoxRun(args, 'box')
+ runs.append(run)
+ print(f'soak: box: {kinds} for {minutes} min, seed {args.seed}, in {run.root}', flush=True)
+ soak(run, kinds, 207, minutes, args.seed)
+ print(table(ROWS))
+ breaches = [(r.cell, b) for r in runs for b in r.breaches]
+ for cell, b in breaches:
+ print(f'BREACH {cell}: {b}')
+ for r in runs:
+ if args.keep or r.breaches:
+ print(f'kept: {r.root}')
+ else:
+ shutil.rmtree(r.root, ignore_errors=True)
+ print('soak:', 'FAIL' if breaches else 'pass')
+ sys.exit(1 if breaches else 0)
+
+
+if __name__ == '__main__':
+ main()