diff options
Diffstat (limited to 'test/delay_relay.py')
| -rw-r--r-- | test/delay_relay.py | 266 |
1 files changed, 266 insertions, 0 deletions
diff --git a/test/delay_relay.py b/test/delay_relay.py new file mode 100644 index 00000000..9d57095f --- /dev/null +++ b/test/delay_relay.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""A link with latency, for the replica gates (docs/next-daemon.md, phase 5): +it accepts on one socket, connects each client on to another, and holds +every chunk `delay` ms each way, timed from its arrival, so a stream's +throughput is not serialised behind the delay and its bytes stay in order. +An optional rate cap makes the link a bottleneck too: a chunk leaves once +the ones before it have crossed at that rate, then waits out the delay. + + python3 -B test/delay_relay.py LISTEN TARGET --delay MS [--rate MB/s] + python3 -B test/delay_relay.py --self-check + +An address is a path (a Unix socket) or host:port (TCP). The one-way +delays the gates use are 15, 40 and 103.5 ms: RTTs of 30, 80 and 207 ms +(the FreeBSD box's). The self-check measures the RTT and throughput it +adds at each, and with a cap. +""" +import argparse +import os +import queue +import socket +import statistics +import sys +import tempfile +import threading +import time + +CHUNK = 256 << 10 + + +def parse(address): + host, sep, port = address.rpartition(':') + if sep and port.isdigit() and '/' not in address: + return socket.AF_INET6 if ':' in host else socket.AF_INET, (host or '127.0.0.1', int(port)) + return socket.AF_UNIX, address + + +def dial(address): + family, addr = parse(address) + s = socket.socket(family) + s.connect(addr) + if family != socket.AF_UNIX: + s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + return s + + +def listen(address): + family, addr = parse(address) + s = socket.socket(family) + if family == socket.AF_UNIX: + try: + os.unlink(addr) + except FileNotFoundError: + pass + else: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(addr) + s.listen(64) + return s + + +def pump(src, dst, delay, rate, done): + """src -> dst, each chunk due `delay` s after it arrived (and after the + ones before it crossed at `rate` bytes/s), in order; its end a shutdown.""" + due_q = queue.SimpleQueue() + + def reader(): + free = 0.0 # when the capped link has sent what it holds + while True: + try: + data = src.recv(CHUNK) + except OSError: + data = b'' + now = time.monotonic() + if rate: + free = max(free, now) + len(data) / rate + now = free + due_q.put((now + delay, data)) + if not data: + return + + def writer(): + try: + while True: + due, data = due_q.get() + wait = due - time.monotonic() + if wait > 0: + time.sleep(wait) + if not data: + dst.shutdown(socket.SHUT_WR) + return + dst.sendall(data) + except OSError: + pass + finally: + done() + + threading.Thread(target=reader, daemon=True).start() + threading.Thread(target=writer, daemon=True).start() + + +def serve(listener, target, delay_ms, rate_mbs=None): + """Relays every connection `listener` accepts until it is closed.""" + delay = delay_ms / 1000 + rate = rate_mbs * 1e6 if rate_mbs else None + while True: + try: + client, _ = listener.accept() + except OSError: + return + try: + upstream = dial(target) + except OSError: + client.close() + continue + if client.family != socket.AF_UNIX: + client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + left = [2] + lock = threading.Lock() + + def done(a=client, b=upstream, left=left, lock=lock): + with lock: + left[0] -= 1 + if left[0] == 0: + a.close() + b.close() + + pump(client, upstream, delay, rate, done) + pump(upstream, client, delay, rate, done) + + +# ---- the self-check ---- + +def echo_server(listener): + def one(c): + with c: + while data := c.recv(CHUNK): + c.sendall(data) + + def loop(): + while True: + try: + c, _ = listener.accept() + except OSError: + return + threading.Thread(target=one, args=(c,), daemon=True).start() + threading.Thread(target=loop, daemon=True).start() + + +def sink_server(listener): + """Reads to the end, then answers one byte: a transfer's end to end.""" + def one(c): + with c: + while c.recv(CHUNK): + pass + c.sendall(b'.') + + def loop(): + while True: + try: + c, _ = listener.accept() + except OSError: + return + threading.Thread(target=one, args=(c,), daemon=True).start() + threading.Thread(target=loop, daemon=True).start() + + +def rtts(address, n=40): + s = dial(address) + out = [] + with s: + for i in range(n): + t = time.monotonic() + s.sendall(i.to_bytes(8, 'little')) + got = b'' + while len(got) < 8: + got += s.recv(8 - len(got)) + out.append((time.monotonic() - t) * 1000) + return out + + +def transfer(address, size): + """Seconds from the first byte sent to the sink's answer.""" + s = dial(address) + block = b'x' * CHUNK + with s: + t = time.monotonic() + left = size + while left > 0: + s.sendall(block[:min(left, CHUNK)]) + left -= CHUNK + s.shutdown(socket.SHUT_WR) + assert s.recv(1) == b'.' + return time.monotonic() - t + + +def self_check(): + tmp = tempfile.mkdtemp(prefix='relay-', dir=os.environ.get('TMPDIR')) + echo, sink = f'{tmp}/echo.sock', f'{tmp}/sink.sock' + echo_server(listen(echo)) + sink_server(listen(sink)) + ok = True + base = statistics.median(rtts(echo)) + size = 256 << 20 + raw = size / transfer(sink, size) / 1e6 + print(f'direct: rtt median {base:.3f} ms, throughput {raw:.0f} MB/s') + for i, delay in enumerate([15, 40, 103.5]): + relays = [] + for name, target in (('e', echo), ('s', sink)): + at = f'{tmp}/{name}{i}.sock' + listener = listen(at) + relays.append(listener) + threading.Thread(target=serve, args=(listener, target, delay), daemon=True).start() + got = rtts(f'{tmp}/e{i}.sock') + lo, med, hi = min(got), statistics.median(got), max(got) + added = med - base + elapsed = transfer(f'{tmp}/s{i}.sock', size) + mbs = size / (elapsed - 2 * delay / 1000) / 1e6 + # The RTT is two delays (scheduling adds well under a millisecond), + # and the delay costs a transfer its two delays, never a delay a + # chunk: serialised, 256 MiB in 256 KiB chunks would take 1024 delays. + good = 2 * delay - 0.05 <= lo and added <= 2 * delay + 1.5 and mbs >= min(200, raw / 4) + ok &= good + print(f'delay {delay} ms each way: rtt min/median/max {lo:.2f}/{med:.2f}/{hi:.2f} ms ' + f'(adds {added:.2f}, want {2 * delay}), throughput {mbs:.0f} MB/s ' + f'({elapsed:.3f} s for {size >> 20} MiB) {"ok" if good else "FAIL"}') + for listener in relays: + listener.close() + # A capped link: the rate holds, and the delay comes on top. + cap, delay, size = 20.0, 40, 40 << 20 + at = f'{tmp}/cap.sock' + listener = listen(at) + threading.Thread(target=serve, args=(listener, sink, delay, cap), daemon=True).start() + elapsed = transfer(at, size) + mbs = size / (elapsed - 2 * delay / 1000) / 1e6 + good = abs(mbs - cap) / cap < 0.05 + ok &= good + print(f'cap {cap} MB/s, delay {delay} ms: {mbs:.2f} MB/s ({elapsed:.3f} s for {size >> 20} MiB) ' + f'{"ok" if good else "FAIL"}') + listener.close() + for f in os.listdir(tmp): + os.unlink(f'{tmp}/{f}') + os.rmdir(tmp) + print('self-check', 'ok' if ok else 'FAILED') + return 0 if ok else 1 + + +def main(): + if '--self-check' in sys.argv: + sys.exit(self_check()) + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('listen') + ap.add_argument('target') + ap.add_argument('--delay', type=float, required=True, help='one-way delay, ms') + ap.add_argument('--rate', type=float, help='a bandwidth cap, MB/s') + a = ap.parse_args() + listener = listen(a.listen) + print(f'relaying {a.listen} -> {a.target}, {a.delay} ms each way' + + (f', {a.rate} MB/s' if a.rate else ''), flush=True) + try: + serve(listener, a.target, a.delay, a.rate) + except KeyboardInterrupt: + pass + + +if __name__ == '__main__': + main() |
