diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/soak.py | 146 |
1 files changed, 128 insertions, 18 deletions
diff --git a/test/soak.py b/test/soak.py index f75371ef..e6626d53 100644 --- a/test/soak.py +++ b/test/soak.py @@ -166,6 +166,12 @@ class Term: self.clears += seen.count(b'\x1b[2J') self.joined_again += (carry + got).count(b'joined again') self.feed(got) + # A rejoin's notice on the screen, counted as it comes up. + for word, attr in ((' diverged from the session', 'shown_diverged'), ('is upgrading', 'shown_upgrade')): + up = any(word in line for line in self.screen.display) + if up and not getattr(self, attr + '_on', False): + setattr(self, attr, getattr(self, attr, 0) + 1) + setattr(self, attr + '_on', up) if self.log: self.log.write(got) if b'\x1b[5n' in seen: @@ -266,6 +272,9 @@ class Base: def env(self): e = dict(self.run.env, PARDES_REPLICA_CHECK=str(self.check), PARDES_REPLICA_STATS=str(self.stats)) + # Its own machine's state (Recent, the store), as a client elsewhere has. + (self.dir / 'state').mkdir(exist_ok=True) + e['XDG_STATE_HOME'] = str(self.dir / 'state') if self.run.args.trace: e['PARDES_REPLICA_TRACE'] = '1' if self.run.far: @@ -362,6 +371,11 @@ class TtyClient(Base): def said_joined(self): return self.term.joined_again + def rejoins(self): + """Joins again: (after a divergence, after an Upgrade's refusal).""" + up = getattr(self.term, 'shown_upgrade', 0) + return max(getattr(self.term, 'shown_diverged', 0), len(self.links()) - 1 - up), up + def hangup(self): """As its terminal closing: the normal way out (its stats written).""" try: @@ -449,6 +463,10 @@ class WinClient(Base): def said_joined(self): return self.said().count('joined again') + def rejoins(self): + said = self.said() + return said.count('joined again: this client diverged'), said.count('joined again: the session is upgrading') + def frame_size(self): try: with open(self.cap / 'latest.ppm', 'rb') as f: @@ -723,6 +741,28 @@ class Run: self.nine = None self.nine_lock = threading.RLock() self.session_pid = None + self.epochs = 0 + self.made = [0] + + def reset_session(self): + """What a dead session left (its daemon, its shells, its sockets) gone, + for the next to start clean.""" + for pid in processes(self.root): + if pid not in [c.p.pid for c in self.clients if c.p and c.p.poll() is None]: + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + time.sleep(0.5) + for f in self.root.glob('pardes-*'): + if f.is_socket() or f.name.endswith('.lock'): + f.unlink(missing_ok=True) + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None def note(self, what): with self.nlock: @@ -920,11 +960,25 @@ class BoxRun(Run): def kill_session(self): self.box(f'env -i {self.benv()} {self.args.remote_bin} --kill={NAME}', timeout=60) + def reset_session(self): + self.kill_remote() + if self.fwd and self.fwd.poll() is None: + self.fwd.kill() + if self.nine: + try: + self.nine.socket.close() + except Exception: + pass + self.nine = None + def relay(self, rtt): return BOX def cleanup(self): super().cleanup() + self.kill_remote() + + def kill_remote(self): # Everything started under run-hunt, by its environment: ps -e prints it. self.box(f'for p in $(ps -axeww -o pid= -o command= | grep "XDG_RUNTIME_DIR={self.run_dir}" | grep -v grep | awk \'{{print $1}}\'); do kill -9 $p; done; ' f'sleep 0.5; rm -rf {self.run_dir}/*', timeout=60) @@ -982,11 +1036,35 @@ def quiet_point(run, live): time.sleep(0.1) if not fresh and c.poll() is None: run.breach(f'{c.label}: freeze: no check within {8 + 2 * c.rtt / 1000:.1f} s at a quiet point') + if fresh: + panes_check(run, c) if isinstance(c, TtyClient) and fresh: view_check(run, c) release(live) +def panes_check(run, c): + """The panes on the client's view (/layout, read while its view is the live + one) against the last quiet point's: one still in the session (/index) and no + longer on its view was lost (a rejoin's new view, a view gone wrong).""" + layout = run.nine_read('/layout') or b'' + index = run.nine_read('/index') or b'' + exist = {int(r.split()[0]) for r in index.splitlines() if r.split() and r.split()[0].isdigit()} + shown = set() + for line in layout.decode(errors='replace').splitlines(): + w = line.split() + if len(w) > 6 and w[0].isdigit(): + shown |= {int(x) for x in w[6:] if x.isdigit()} + before = getattr(c, 'shown', None) + c.shown = shown + if before is None: + return + lost = (before - shown) & exist + if lost: + c.lost = getattr(c, 'lost', 0) + len(lost) + run.breach(f'{c.label}: panes {sorted(lost)} left its view while still open (links {len(c.links())})') + + def view_check(run, c): """The terminal client's screen and cursor against what the session renders for the same view (its input was the last, so 9P reads that view).""" @@ -1084,23 +1162,32 @@ def global_event(run, rng, live, made): def soak(run, kinds, rtt, minutes, seed): + """The cell: its minutes of typing, the session started again (and every + client) whenever it dies, as its user would, so a cell measures to its end.""" rng = random.Random(seed) - stop = threading.Event() - live = [] - threads = [] - made = [0] + end = now() + minutes * 60 try: - soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made) + while True: + run.epochs += 1 + died = soak_body(run, kinds, rtt, end, rng) + if not died or now() > end - 30 or run.epochs >= 20 or not run.args.respawn: + break + run.note(f'the session started again (epoch {run.epochs + 1})') + run.reset_session() except Exception: import traceback run.breach('harness: ' + traceback.format_exc().strip().replace('\n', ' | ')[-600:]) finally: - stop.set() report_cell(run, rtt) run.cleanup() -def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): +def soak_body(run, kinds, rtt, end, rng): + """One session's life in the cell: True if it died before `end`.""" + stop = threading.Event() + live = [] + threads = [] + made = run.made run.start_session() for kind in kinds: made[0] += 1 @@ -1109,7 +1196,6 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): t = threading.Thread(target=monkey, args=(c, random.Random(rng.random()), stop), daemon=True) t.start() threads.append(t) - end = now() + minutes * 60 next_quiet = now() + 45 next_event = now() + rng.uniform(20, 60) while now() < end: @@ -1118,9 +1204,31 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): if c.poll() is not None: run.breach(f'{c.label}: exited {c.gone} unasked: {tail(c)}') live.remove(c) + c.ended_at = now() + if run.session_alive() and run.args.respawn: + # As its user would: open it again, and go on. + made[0] += 1 + try: + n = start_client(run, c.kind, c.rtt, made[0]) + except TimeoutError as e: + run.breach(f'{c.kind}: a client started again gave no first frame: {e}') + continue + live.append(n) + t = threading.Thread(target=monkey, args=(n, random.Random(rng.random()), stop), daemon=True) + t.start() + threads.append(t) if not run.session_alive(): - run.breach('the session died') - break + run.breach(f'the session died (epoch {run.epochs}, {now() - run.t0:.0f} s into the cell)') + stop.set() + for c in live: + c.asked_to_end = True + for c in live: + try: + wait_for(f'{c.label}\'s end with its session', lambda: c.poll() is not None, timeout=10) + except TimeoutError: + run.breach(f'{c.label}: still running 10 s after its session died') + c.p.kill() + return True if not live: break if now() > next_event: @@ -1151,6 +1259,7 @@ def soak_body(run, kinds, rtt, minutes, rng, stop, live, threads, made): c.p.kill() if run.session_alive(): run.end_session() + return False def tail(c): @@ -1194,16 +1303,16 @@ def report_cell(run, rtt): links = len(c.links()) lines = c.check_lines() differ = [line for line in lines if line.startswith('differ')] - said = c.said_joined() - if links > 1 or said: - run.breach(f'{c.label}: rejoined (links {links}, "joined again" {said})') + diverged, upgraded = c.rejoins() + if diverged: + run.breach(f'{c.label}: rejoined {diverged} times after a divergence (links {links}, {upgraded} after an Upgrade)') if differ: run.breach(f'{c.label}: {len(differ)} checks differ') if c.gone is not None and c.gone < 0 and c.gone not in (-signal.SIGHUP, -signal.SIGKILL): run.breach(f'{c.label}: died of signal {-c.gone}') if c.gone not in (None, 0) and not c.asked_to_end: pass # said when found - elif c.gone not in (None, 0, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: + elif c.gone not in (None, 0, 1, -signal.SIGHUP, -signal.SIGKILL) and c.asked_to_end: run.breach(f'{c.label}: ended {c.gone} after a detach or hangup: {tail(c)}') joined = c.said() for word in ('panic', 'reached unreachable', 'Segmentation', 'General protection', 'aborting'): @@ -1216,11 +1325,11 @@ def report_cell(run, rtt): lat, missed = c.lat, c.no_output dur = max(1e-6, ((c.ended_at if hasattr(c, 'ended_at') else now()) - c.started) / 60) ROWS.append({ - 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, + 'cell': run.cell, 'client': c.label, 'rtt': rtt, 'min': dur, 'inputs': c.inputs, 'links': links, 'rejoins': diverged, 'checks': len(lines), 'differ': len(differ), 'visible': st['visible'], 'mispred': st['mispredicted'], 'vis/min': st['visible'] / dur, 'clears/min': (c.term.clears / dur) if isinstance(c, TtyClient) else None, 'p50': pct(lat, 50), 'p99': pct(lat, 99), 'nolat': missed, 'views': getattr(c, 'views', 0), - 'view_bad': getattr(c, 'view_bad', 0), 'exit': c.gone, + 'view_bad': getattr(c, 'view_bad', 0), 'lost': getattr(c, 'lost', 0), 'exit': c.gone, 'epochs': run.epochs, }) st = (run.root / 'session.stats') text = '' @@ -1241,8 +1350,8 @@ def report_cell(run, rtt): def table(rows): - cols = ['cell', 'client', 'min', 'inputs', 'links', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', - 'p50', 'p99', 'nolat', 'views', 'view_bad', 'exit'] + cols = ['cell', 'client', 'min', 'inputs', 'links', 'rejoins', 'checks', 'differ', 'visible', 'mispred', 'vis/min', 'clears/min', + 'p50', 'p99', 'nolat', 'views', 'view_bad', 'lost', 'exit', 'epochs'] def fmt(v): if v is None: return '-' @@ -1277,6 +1386,7 @@ def main(): ap.add_argument('--pardes', default=os.path.expanduser('~/.local/bin/pardes')) ap.add_argument('--gui', default=os.path.expanduser('~/.local/bin/pardes-gui')) ap.add_argument('--remote-bin', default=f'{BOX_T}/bin/pardes-hunt') + ap.add_argument('--no-respawn', dest='respawn', action='store_false', help='a client that dies stays dead') ap.add_argument('--trace', action='store_true', help='PARDES_REPLICA_TRACE on the session: per-field digests') ap.add_argument('--no-upgrade', dest='upgrade', action='store_false') ap.add_argument('--near', action='store_true', help='local clients near (no zstd, no blobs by hash)') |
