#!/usr/bin/env python3 """A link with latency, for the replica gates (docs/next-daemon.md, phase 5): it accepts on one socket, connects each client on to another, and holds every chunk `delay` ms each way, timed from its arrival, so a stream's throughput is not serialised behind the delay and its bytes stay in order. An optional rate cap makes the link a bottleneck too: a chunk leaves once the ones before it have crossed at that rate, then waits out the delay. python3 -B test/delay_relay.py LISTEN TARGET --delay MS [--rate MB/s] python3 -B test/delay_relay.py --self-check An address is a path (a Unix socket) or host:port (TCP). The one-way delays the gates use are 15, 40 and 103.5 ms: RTTs of 30, 80 and 207 ms (the FreeBSD box's). The self-check measures the RTT and throughput it adds at each, and with a cap. """ import argparse import os import queue import socket import statistics import sys import tempfile import threading import time CHUNK = 256 << 10 def parse(address): host, sep, port = address.rpartition(':') if sep and port.isdigit() and '/' not in address: return socket.AF_INET6 if ':' in host else socket.AF_INET, (host or '127.0.0.1', int(port)) return socket.AF_UNIX, address def dial(address): family, addr = parse(address) s = socket.socket(family) s.connect(addr) if family != socket.AF_UNIX: s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) return s def listen(address): family, addr = parse(address) s = socket.socket(family) if family == socket.AF_UNIX: try: os.unlink(addr) except FileNotFoundError: pass else: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(addr) s.listen(64) return s def pump(src, dst, delay, rate, done): """src -> dst, each chunk due `delay` s after it arrived (and after the ones before it crossed at `rate` bytes/s), in order; its end a shutdown.""" due_q = queue.SimpleQueue() def reader(): free = 0.0 # when the capped link has sent what it holds while True: try: data = src.recv(CHUNK) except OSError: data = b'' now = time.monotonic() if rate: free = max(free, now) + len(data) / rate now = free due_q.put((now + delay, data)) if not data: return def writer(): try: while True: due, data = due_q.get() wait = due - time.monotonic() if wait > 0: time.sleep(wait) if not data: dst.shutdown(socket.SHUT_WR) return dst.sendall(data) except OSError: pass finally: done() threading.Thread(target=reader, daemon=True).start() threading.Thread(target=writer, daemon=True).start() def serve(listener, target, delay_ms, rate_mbs=None): """Relays every connection `listener` accepts until it is closed.""" delay = delay_ms / 1000 rate = rate_mbs * 1e6 if rate_mbs else None while True: try: client, _ = listener.accept() except OSError: return try: upstream = dial(target) except OSError: client.close() continue if client.family != socket.AF_UNIX: client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) left = [2] lock = threading.Lock() def done(a=client, b=upstream, left=left, lock=lock): with lock: left[0] -= 1 if left[0] == 0: a.close() b.close() pump(client, upstream, delay, rate, done) pump(upstream, client, delay, rate, done) # ---- the self-check ---- def echo_server(listener): def one(c): with c: while data := c.recv(CHUNK): c.sendall(data) def loop(): while True: try: c, _ = listener.accept() except OSError: return threading.Thread(target=one, args=(c,), daemon=True).start() threading.Thread(target=loop, daemon=True).start() def sink_server(listener): """Reads to the end, then answers one byte: a transfer's end to end.""" def one(c): with c: while c.recv(CHUNK): pass c.sendall(b'.') def loop(): while True: try: c, _ = listener.accept() except OSError: return threading.Thread(target=one, args=(c,), daemon=True).start() threading.Thread(target=loop, daemon=True).start() def rtts(address, n=40): s = dial(address) out = [] with s: for i in range(n): t = time.monotonic() s.sendall(i.to_bytes(8, 'little')) got = b'' while len(got) < 8: got += s.recv(8 - len(got)) out.append((time.monotonic() - t) * 1000) return out def transfer(address, size): """Seconds from the first byte sent to the sink's answer.""" s = dial(address) block = b'x' * CHUNK with s: t = time.monotonic() left = size while left > 0: s.sendall(block[:min(left, CHUNK)]) left -= CHUNK s.shutdown(socket.SHUT_WR) assert s.recv(1) == b'.' return time.monotonic() - t def self_check(): tmp = tempfile.mkdtemp(prefix='relay-', dir=os.environ.get('TMPDIR')) echo, sink = f'{tmp}/echo.sock', f'{tmp}/sink.sock' echo_server(listen(echo)) sink_server(listen(sink)) ok = True base = statistics.median(rtts(echo)) size = 256 << 20 raw = size / transfer(sink, size) / 1e6 print(f'direct: rtt median {base:.3f} ms, throughput {raw:.0f} MB/s') for i, delay in enumerate([15, 40, 103.5]): relays = [] for name, target in (('e', echo), ('s', sink)): at = f'{tmp}/{name}{i}.sock' listener = listen(at) relays.append(listener) threading.Thread(target=serve, args=(listener, target, delay), daemon=True).start() got = rtts(f'{tmp}/e{i}.sock') lo, med, hi = min(got), statistics.median(got), max(got) added = med - base elapsed = transfer(f'{tmp}/s{i}.sock', size) mbs = size / (elapsed - 2 * delay / 1000) / 1e6 # The RTT is two delays (scheduling adds well under a millisecond), # and the delay costs a transfer its two delays, never a delay a # chunk: serialised, 256 MiB in 256 KiB chunks would take 1024 delays. good = 2 * delay - 0.05 <= lo and added <= 2 * delay + 1.5 and mbs >= min(200, raw / 4) ok &= good print(f'delay {delay} ms each way: rtt min/median/max {lo:.2f}/{med:.2f}/{hi:.2f} ms ' f'(adds {added:.2f}, want {2 * delay}), throughput {mbs:.0f} MB/s ' f'({elapsed:.3f} s for {size >> 20} MiB) {"ok" if good else "FAIL"}') for listener in relays: listener.close() # A capped link: the rate holds, and the delay comes on top. cap, delay, size = 20.0, 40, 40 << 20 at = f'{tmp}/cap.sock' listener = listen(at) threading.Thread(target=serve, args=(listener, sink, delay, cap), daemon=True).start() elapsed = transfer(at, size) mbs = size / (elapsed - 2 * delay / 1000) / 1e6 good = abs(mbs - cap) / cap < 0.05 ok &= good print(f'cap {cap} MB/s, delay {delay} ms: {mbs:.2f} MB/s ({elapsed:.3f} s for {size >> 20} MiB) ' f'{"ok" if good else "FAIL"}') listener.close() for f in os.listdir(tmp): os.unlink(f'{tmp}/{f}') os.rmdir(tmp) print('self-check', 'ok' if ok else 'FAILED') return 0 if ok else 1 def main(): if '--self-check' in sys.argv: sys.exit(self_check()) ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) ap.add_argument('listen') ap.add_argument('target') ap.add_argument('--delay', type=float, required=True, help='one-way delay, ms') ap.add_argument('--rate', type=float, help='a bandwidth cap, MB/s') a = ap.parse_args() listener = listen(a.listen) print(f'relaying {a.listen} -> {a.target}, {a.delay} ms each way' + (f', {a.rate} MB/s' if a.rate else ''), flush=True) try: serve(listener, a.target, a.delay, a.rate) except KeyboardInterrupt: pass if __name__ == '__main__': main()