summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--test/soak.py146
1 files changed, 128 insertions, 18 deletions
diff --git a/test/soak.py b/test/soak.py
index f75371ef..e6626d53 100644
--- a/test/soak.py
+++ b/test/soak.py
@@ -166,6 +166,12 @@ class Term:
self.clears += seen.count(b'\x1b[2J')
self.joined_again += (carry + got).count(b'joined again')
self.feed(got)
+ # A rejoin's notice on the screen, counted as it comes up.
+ for word, attr in ((' diverged from the session', 'shown_diverged'), ('is upgrading', 'shown_upgrade')):
+ up = any(word in line for line in self.screen.display)
+ if up and not getattr(self, attr + '_on', False):
+ setattr(self, attr, getattr(self, attr, 0) + 1)
+ setattr(self, attr + '_on', up)
if self.log:
self.log.write(got)
if b'\x1b[5n' in seen:
@@ -266,6 +272,9 @@ class Base:
def env(self):
e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats))
+ # Its own machine's state (Recent, the store), as a client elsewhere has.
+ (self.dir / 'state').mkdir(exist_ok=True)
+ e['XDG_STATE_HOME'] = str(self.dir / 'state')
if self.run.args.trace:
e['PARDES_REPLICA_TRACE'] = '1'
if self.run.far:
@@ -362,6 +371,11 @@ class TtyClient(Base):
def said_joined(self):
return self.term.joined_again
+ def rejoins(self):
+ """Joins again: (after a divergence, after an Upgrade's refusal)."""
+ up = getattr(self.term, 'shown_upgrade', 0)
+ return max(getattr(self.term, 'shown_diverged', 0), len(self.links()) - 1 - up), up
+
def hangup(self):
"""As its terminal closing: the normal way out (its stats written)."""
try:
@@ -449,6 +463,10 @@ class WinClient(Base):
def said_joined(self):
return self.said().count('joined again')
+ def rejoins(self):
+ said = self.said()
+ return said.count('joined again: this client diverged'), said.count('joined again: the session is upgrading')
+
def frame_size(self):
try:
with open(self.cap / 'latest.ppm', 'rb') as f:
@@ -723,6 +741,28 @@ class Run:
self.nine = None
self.nine_lock = threading.RLock()
self.session_pid = None
+ self.epochs = 0
+ self.made = [0]
+
+ def reset_session(self):
+ """What a dead session left (its daemon, its shells, its sockets) gone,
+ for the next to start clean."""
+ for pid in processes(self.root):
+ if pid not in [c.p.pid for c in self.clients if c.p and c.p.poll() is None]:
+ try:
+ os.kill(pid, signal.SIGKILL)
+ except OSError:
+ pass
+ time.sleep(0.5)
+ for f in self.root.glob('pardes-*'):
+ if f.is_socket() or f.name.endswith('.lock'):
+ f.unlink(missing_ok=True)
+ if self.nine:
+ try:
+ self.nine.socket.close()
+ except Exception:
+ pass
+ self.nine = None
def note(self, what):
with self.nlock:
@@ -920,11 +960,25 @@ class BoxRun(Run):
def kill_session(self):
self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60)
+ def reset_session(self):
+ self.kill_remote()
+ if self.fwd and self.fwd.poll() is None:
+ self.fwd.kill()
+ if self.nine:
+ try:
+ self.nine.socket.close()
+ except Exception:
+ pass
+ self.nine = None
+
def relay(self, rtt):
return BOX
def cleanup(self):
super().cleanup()
+ self.kill_remote()
+
+ def kill_remote(self):
# Everything started under run-hunt, by its environment: ps -e prints it.
self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; '
f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60)
@@ -982,11 +1036,35 @@ def quiet_point(run, live):
time.sleep(0.1)
if not fresh and c.poll() is None:
run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point')
+ if fresh:
+ panes_check(run, c)
if isinstance(c, TtyClient) and fresh:
view_check(run, c)
release(live)
+def panes_check(run, c):
+ """The panes on the client's view (/layout, read while its view is the live
+ one) against the last quiet point's: one still in the session (/index) and no
+ longer on its view was lost (a rejoin's new view, a view gone wrong)."""
+ layout = run.nine_read('/layout') or b''
+ index = run.nine_read('/index') or b''
+ exist = {int(r.split()[0]) for r in index.splitlines() if r.split() and r.split()[0].isdigit()}
+ shown = set()
+ for line in layout.decode(errors='replace').splitlines():
+ w = line.split()
+ if len(w) > 6 and w[0].isdigit():
+ shown |= {int(x) for x in w[6:] if x.isdigit()}
+ before = getattr(c, 'shown', None)
+ c.shown = shown
+ if before is None:
+ return
+ lost = (before - shown) & exist
+ if lost:
+ c.lost = getattr(c, 'lost', 0) + len(lost)
+ run.breach(f'{c.label}: panes {sorted(lost)} left its view while still open (links {len(c.links())})')
+
+
def view_check(run, c):
"""The terminal client's screen and cursor against what the session renders
for the same view (its input was the last, so 9P reads that view)."""
@@ -1084,23 +1162,32 @@ def global_event(run, rng, live, made):
def soak(run, kinds, rtt, minutes, seed):
+ """The cell: its minutes of typing, the session started again (and every
+ client) whenever it dies, as its user would, so a cell measures to its end."""
rng = random.Random(seed)
- stop = threading.Event()
- live = []
- threads = []
- made = [0]
+ end = now() + minutes * 60
try:
- soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made)
+ while True:
+ run.epochs += 1
+ died = soak_body(run, kinds, rtt, end, rng)
+ if not died or now() > end - 30 or run.epochs >= 20 or not run.args.respawn:
+ break
+ run.note(f'the session started again (epoch {run.epochs + 1})')
+ run.reset_session()
except Exception:
import traceback
run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:])
finally:
- stop.set()
report_cell(run, rtt)
run.cleanup()
-def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made):
+def soak_body(run, kinds, rtt, end, rng):
+ """One session's life in the cell: True if it died before `end`."""
+ stop = threading.Event()
+ live = []
+ threads = []
+ made = run.made
run.start_session()
for kind in kinds:
made[0] += 1
@@ -1109,7 +1196,6 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made):
t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True)
t.start()
threads.append(t)
- end = now() + minutes * 60
next_quiet = now() + 45
next_event = now() + rng.uniform(20, 60)
while now() < end:
@@ -1118,9 +1204,31 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made):
if c.poll() is not None:
run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}')
live.remove(c)
+ c.ended_at = now()
+ if run.session_alive() and run.args.respawn:
+ # As its user would: open it again, and go on.
+ made[0] += 1
+ try:
+ n = start_client(run, c.kind, c.rtt, made[0])
+ except TimeoutError as e:
+ run.breach(f'{c.kind}: a client started again gave no first frame: {e}')
+ continue
+ live.append(n)
+ t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True)
+ t.start()
+ threads.append(t)
if not run.session_alive():
- run.breach('the session died')
- break
+ run.breach(f'the session died (epoch {run.epochs}, {now() - run.t0:.0f} s into the cell)')
+ stop.set()
+ for c in live:
+ c.asked_to_end = True
+ for c in live:
+ try:
+ wait_for(f'{c.label}\'s end with its session', lambda: c.poll() is not None, timeout=10)
+ except TimeoutError:
+ run.breach(f'{c.label}: still running 10 s after its session died')
+ c.p.kill()
+ return True
if not live:
break
if now() > next_event:
@@ -1151,6 +1259,7 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made):
c.p.kill()
if run.session_alive():
run.end_session()
+ return False
def tail(c):
@@ -1194,16 +1303,16 @@ def report_cell(run, rtt):
links = len(c.links())
lines = c.check_lines()
differ = [line for line in lines if line.startswith('differ')]
- said = c.said_joined()
- if links > 1 or said:
- run.breach(f'{c.label}: rejoined (links {links}, "joined again" {said})')
+ diverged, upgraded = c.rejoins()
+ if diverged:
+ run.breach(f'{c.label}: rejoined {diverged} times after a divergence (links {links}, {upgraded} after an Upgrade)')
if differ:
run.breach(f'{c.label}: {len(differ)} checks differ')
if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL):
run.breach(f'{c.label}: died of signal {-c.gone}')
if c.gone not in (None, 0) and not c.asked_to_end:
pass # said when found
- elif c.gone not in (None, 0, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end:
+ elif c.gone not in (None, 0, 1, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end:
run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}')
joined = c.said()
for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'):
@@ -1216,11 +1325,11 @@ def report_cell(run, rtt):
lat, missed = c.lat, c.no_output
dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60)
ROWS.append({
- 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links,
+ 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, 'rejoins': diverged,
'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'],
'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None,
'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0),
- 'view_bad': getattr(c, 'view_bad', 0), 'exit': c.gone,
+ 'view_bad': getattr(c, 'view_bad', 0), 'lost': getattr(c, 'lost', 0), 'exit': c.gone, 'epochs': run.epochs,
})
st = (run.root / 'session.stats')
text = ''
@@ -1241,8 +1350,8 @@ def report_cell(run, rtt):
def table(rows):
- cols = ['cell', 'client', 'min', 'inputs', 'links', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min',
- 'p50', 'p99', 'nolat', 'views', 'view_bad', 'exit']
+ cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min',
+ 'p50', 'p99', 'nolat', 'views', 'view_bad', 'lost', 'exit', 'epochs']
def fmt(v):
if v is None:
return '-'
@@ -1277,6 +1386,7 @@ def main():
ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes'))
ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui'))
ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt')
+ ap.add_argument('--no-respawn', dest='respawn', action='store_false', help='a client that dies stays dead')
ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests')
ap.add_argument('--no-upgrade', dest='upgrade', action='store_false')
ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)')