#!/usr/bin/env python3 """A link with latency, for the replica gates (docs/next-daemon.md, phase 5): it accepts on one socket, connects each client on to another, and holds every chunk `delay` ms each way, timed from its arrival, so a stream's throughput is not serialised behind the delay and its bytes stay in order. An optional rate cap makes the link a bottleneck too: a chunk leaves once the ones before it have crossed at that rate, then waits out the delay. An optional jitter adds up to that many ms more to each chunk's delay, at random, its order kept (a chunk never overtakes the one before it). python3 -B test/delay_relay.py LISTEN TARGET --delay MS [--rate MB/s] [--jitter MS] python3 -B test/delay_relay.py --self-check An address is a path (a Unix socket) or host:port (TCP). The one-way delays the gates use are 15, 40 and 103.5 ms: RTTs of 30, 80 and 207 ms (the FreeBSD box's). The self-check measures the RTT and throughput it adds at each, and with a cap. """ import argparse import os import queue import random import socket import statistics import sys import tempfile import threading import time CHUNK = 256 << 10 def parse(address): host, sep, port = address.rpartition(':') if sep and port.isdigit() and '/' not in address: return socket.AF_INET6 if ':' in host else socket.AF_INET, (host or '127.0.0.1', int(port)) return socket.AF_UNIX, address def dial(address): family, addr = parse(address) s = socket.socket(family) s.connect(addr) if family != socket.AF_UNIX: s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) return s def listen(address): family, addr = parse(address) s = socket.socket(family) if family == socket.AF_UNIX: try: os.unlink(addr) except FileNotFoundError: pass else: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind(addr) s.listen(64) return s def pump(src, dst, delay, rate, done, jitter=0.0): """src -> dst, each chunk due `delay` s after it arrived (and after the ones before it crossed at `rate` bytes/s), plus up to `jitter` s at random but never before the chunk ahead of it; its end a shutdown.""" due_q = queue.SimpleQueue() def reader(): free = 0.0 # when the capped link has sent what it holds last = 0.0 # when the chunk before is due: order is kept while True: try: data = src.recv(CHUNK) except OSError: data = b'' now = time.monotonic() if rate: free = max(free, now) + len(data) / rate now = free last = max(last, now + delay + (random.uniform(0, jitter) if jitter else 0)) due_q.put((last, data)) if not data: return def writer(): try: while True: due, data = due_q.get() wait = due - time.monotonic() if wait > 0: time.sleep(wait) if not data: dst.shutdown(socket.SHUT_WR) return dst.sendall(data) except OSError: pass finally: done() threading.Thread(target=reader, daemon=True).start() threading.Thread(target=writer, daemon=True).start() def serve(listener, target, delay_ms, rate_mbs=None, jitter_ms=0): """Relays every connection `listener` accepts until it is closed.""" delay = delay_ms / 1000 jitter = jitter_ms / 1000 rate = rate_mbs * 1e6 if rate_mbs else None while True: try: client, _ = listener.accept() except OSError: return try: upstream = dial(target) except OSError: client.close() continue if client.family != socket.AF_UNIX: client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) left = [2] lock = threading.Lock() def done(a=client, b=upstream, left=left, lock=lock): with lock: left[0] -= 1 if left[0] == 0: a.close() b.close() pump(client, upstream, delay, rate, done, jitter) pump(upstream, client, delay, rate, done, jitter) # ---- the self-check ---- def echo_server(listener): def one(c): with c: while data := c.recv(CHUNK): c.sendall(data) def loop(): while True: try: c, _ = listener.accept() except OSError: return threading.Thread(target=one, args=(c,), daemon=True).start() threading.Thread(target=loop, daemon=True).start() def sink_server(listener): """Reads to the end, then answers one byte: a transfer's end to end.""" def one(c): with c: while c.recv(CHUNK): pass c.sendall(b'.') def loop(): while True: try: c, _ = listener.accept() except OSError: return threading.Thread(target=one, args=(c,), daemon=True).start() threading.Thread(target=loop, daemon=True).start() def rtts(address, n=40): s = dial(address) out = [] with s: for i in range(n): t = time.monotonic() s.sendall(i.to_bytes(8, 'little')) got = b'' while len(got) < 8: got += s.recv(8 - len(got)) out.append((time.monotonic() - t) * 1000) return out def transfer(address, size): """Seconds from the first byte sent to the sink's answer.""" s = dial(address) block = b'x' * CHUNK with s: t = time.monotonic() left = size while left > 0: s.sendall(block[:min(left, CHUNK)]) left -= CHUNK s.shutdown(socket.SHUT_WR) assert s.recv(1) == b'.' return time.monotonic() - t def self_check(): tmp = tempfile.mkdtemp(prefix='relay-', dir=os.environ.get('TMPDIR')) echo, sink = f'{tmp}/echo.sock', f'{tmp}/sink.sock' echo_server(listen(echo)) sink_server(listen(sink)) ok = True base = statistics.median(rtts(echo)) size = 256 << 20 raw = size / transfer(sink, size) / 1e6 print(f'direct: rtt median {base:.3f} ms, throughput {raw:.0f} MB/s') for i, delay in enumerate([15, 40, 103.5]): relays = [] for name, target in (('e', echo), ('s', sink)): at = f'{tmp}/{name}{i}.sock' listener = listen(at) relays.append(listener) threading.Thread(target=serve, args=(listener, target, delay), daemon=True).start() got = rtts(f'{tmp}/e{i}.sock') lo, med, hi = min(got), statistics.median(got), max(got) added = med - base elapsed = transfer(f'{tmp}/s{i}.sock', size) mbs = size / (elapsed - 2 * delay / 1000) / 1e6 # The RTT is two delays (scheduling adds well under a millisecond), # and the delay costs a transfer its two delays, never a delay a # chunk: serialised, 256 MiB in 256 KiB chunks would take 1024 delays. good = 2 * delay - 0.05 <= lo and added <= 2 * delay + 1.5 and mbs >= min(200, raw / 4) ok &= good print(f'delay {delay} ms each way: rtt min/median/max {lo:.2f}/{med:.2f}/{hi:.2f} ms ' f'(adds {added:.2f}, want {2 * delay}), throughput {mbs:.0f} MB/s ' f'({elapsed:.3f} s for {size >> 20} MiB) {"ok" if good else "FAIL"}') for listener in relays: listener.close() # A capped link: the rate holds, and the delay comes on top. cap, delay, size = 20.0, 40, 40 << 20 at = f'{tmp}/cap.sock' listener = listen(at) threading.Thread(target=serve, args=(listener, sink, delay, cap), daemon=True).start() elapsed = transfer(at, size) mbs = size / (elapsed - 2 * delay / 1000) / 1e6 good = abs(mbs - cap) / cap < 0.05 ok &= good print(f'cap {cap} MB/s, delay {delay} ms: {mbs:.2f} MB/s ({elapsed:.3f} s for {size >> 20} MiB) ' f'{"ok" if good else "FAIL"}') listener.close() for f in os.listdir(tmp): os.unlink(f'{tmp}/{f}') os.rmdir(tmp) print('self-check', 'ok' if ok else 'FAILED') return 0 if ok else 1 def main(): if '--self-check' in sys.argv: sys.exit(self_check()) ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) ap.add_argument('listen') ap.add_argument('target') ap.add_argument('--delay', type=float, required=True, help='one-way delay, ms') ap.add_argument('--rate', type=float, help='a bandwidth cap, MB/s') ap.add_argument('--jitter', type=float, default=0, help='up to this many ms more a chunk, at random') a = ap.parse_args() listener = listen(a.listen) print(f'relaying {a.listen} -> {a.target}, {a.delay} ms each way' + (f', {a.rate} MB/s' if a.rate else '') + (f', jitter {a.jitter} ms' if a.jitter else ''), flush=True) try: serve(listener, a.target, a.delay, a.rate, a.jitter) except KeyboardInterrupt: pass if __name__ == '__main__': main()