summaryrefslogtreecommitdiff
path: root/test/delay_relay.py
diff options
context:
space:
mode:
Diffstat (limited to 'test/delay_relay.py')
-rw-r--r--test/delay_relay.py266
1 files changed, 266 insertions, 0 deletions
diff --git a/test/delay_relay.py b/test/delay_relay.py
new file mode 100644
index 00000000..9d57095f
--- /dev/null
+++ b/test/delay_relay.py
@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+"""A link with latency, for the replica gates (docs/next-daemon.md, phase 5):
+it accepts on one socket, connects each client on to another, and holds
+every chunk `delay` ms each way, timed from its arrival, so a stream's
+throughput is not serialised behind the delay and its bytes stay in order.
+An optional rate cap makes the link a bottleneck too: a chunk leaves once
+the ones before it have crossed at that rate, then waits out the delay.
+
+ python3 -B test/delay_relay.py LISTEN TARGET --delay MS [--rate MB/s]
+ python3 -B test/delay_relay.py --self-check
+
+An address is a path (a Unix socket) or host:port (TCP). The one-way
+delays the gates use are 15, 40 and 103.5 ms: RTTs of 30, 80 and 207 ms
+(the FreeBSD box's). The self-check measures the RTT and throughput it
+adds at each, and with a cap.
+"""
+import argparse
+import os
+import queue
+import socket
+import statistics
+import sys
+import tempfile
+import threading
+import time
+
+CHUNK = 256 << 10
+
+
+def parse(address):
+ host, sep, port = address.rpartition(':')
+ if sep and port.isdigit() and '/' not in address:
+ return socket.AF_INET6 if ':' in host else socket.AF_INET, (host or '127.0.0.1', int(port))
+ return socket.AF_UNIX, address
+
+
+def dial(address):
+ family, addr = parse(address)
+ s = socket.socket(family)
+ s.connect(addr)
+ if family != socket.AF_UNIX:
+ s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ return s
+
+
+def listen(address):
+ family, addr = parse(address)
+ s = socket.socket(family)
+ if family == socket.AF_UNIX:
+ try:
+ os.unlink(addr)
+ except FileNotFoundError:
+ pass
+ else:
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ s.bind(addr)
+ s.listen(64)
+ return s
+
+
+def pump(src, dst, delay, rate, done):
+ """src -> dst, each chunk due `delay` s after it arrived (and after the
+ ones before it crossed at `rate` bytes/s), in order; its end a shutdown."""
+ due_q = queue.SimpleQueue()
+
+ def reader():
+ free = 0.0 # when the capped link has sent what it holds
+ while True:
+ try:
+ data = src.recv(CHUNK)
+ except OSError:
+ data = b''
+ now = time.monotonic()
+ if rate:
+ free = max(free, now) + len(data) / rate
+ now = free
+ due_q.put((now + delay, data))
+ if not data:
+ return
+
+ def writer():
+ try:
+ while True:
+ due, data = due_q.get()
+ wait = due - time.monotonic()
+ if wait > 0:
+ time.sleep(wait)
+ if not data:
+ dst.shutdown(socket.SHUT_WR)
+ return
+ dst.sendall(data)
+ except OSError:
+ pass
+ finally:
+ done()
+
+ threading.Thread(target=reader, daemon=True).start()
+ threading.Thread(target=writer, daemon=True).start()
+
+
+def serve(listener, target, delay_ms, rate_mbs=None):
+ """Relays every connection `listener` accepts until it is closed."""
+ delay = delay_ms / 1000
+ rate = rate_mbs * 1e6 if rate_mbs else None
+ while True:
+ try:
+ client, _ = listener.accept()
+ except OSError:
+ return
+ try:
+ upstream = dial(target)
+ except OSError:
+ client.close()
+ continue
+ if client.family != socket.AF_UNIX:
+ client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ left = [2]
+ lock = threading.Lock()
+
+ def done(a=client, b=upstream, left=left, lock=lock):
+ with lock:
+ left[0] -= 1
+ if left[0] == 0:
+ a.close()
+ b.close()
+
+ pump(client, upstream, delay, rate, done)
+ pump(upstream, client, delay, rate, done)
+
+
+# ---- the self-check ----
+
+def echo_server(listener):
+ def one(c):
+ with c:
+ while data := c.recv(CHUNK):
+ c.sendall(data)
+
+ def loop():
+ while True:
+ try:
+ c, _ = listener.accept()
+ except OSError:
+ return
+ threading.Thread(target=one, args=(c,), daemon=True).start()
+ threading.Thread(target=loop, daemon=True).start()
+
+
+def sink_server(listener):
+ """Reads to the end, then answers one byte: a transfer's end to end."""
+ def one(c):
+ with c:
+ while c.recv(CHUNK):
+ pass
+ c.sendall(b'.')
+
+ def loop():
+ while True:
+ try:
+ c, _ = listener.accept()
+ except OSError:
+ return
+ threading.Thread(target=one, args=(c,), daemon=True).start()
+ threading.Thread(target=loop, daemon=True).start()
+
+
+def rtts(address, n=40):
+ s = dial(address)
+ out = []
+ with s:
+ for i in range(n):
+ t = time.monotonic()
+ s.sendall(i.to_bytes(8, 'little'))
+ got = b''
+ while len(got) < 8:
+ got += s.recv(8 - len(got))
+ out.append((time.monotonic() - t) * 1000)
+ return out
+
+
+def transfer(address, size):
+ """Seconds from the first byte sent to the sink's answer."""
+ s = dial(address)
+ block = b'x' * CHUNK
+ with s:
+ t = time.monotonic()
+ left = size
+ while left > 0:
+ s.sendall(block[:min(left, CHUNK)])
+ left -= CHUNK
+ s.shutdown(socket.SHUT_WR)
+ assert s.recv(1) == b'.'
+ return time.monotonic() - t
+
+
+def self_check():
+ tmp = tempfile.mkdtemp(prefix='relay-', dir=os.environ.get('TMPDIR'))
+ echo, sink = f'{tmp}/echo.sock', f'{tmp}/sink.sock'
+ echo_server(listen(echo))
+ sink_server(listen(sink))
+ ok = True
+ base = statistics.median(rtts(echo))
+ size = 256 << 20
+ raw = size / transfer(sink, size) / 1e6
+ print(f'direct: rtt median {base:.3f} ms, throughput {raw:.0f} MB/s')
+ for i, delay in enumerate([15, 40, 103.5]):
+ relays = []
+ for name, target in (('e', echo), ('s', sink)):
+ at = f'{tmp}/{name}{i}.sock'
+ listener = listen(at)
+ relays.append(listener)
+ threading.Thread(target=serve, args=(listener, target, delay), daemon=True).start()
+ got = rtts(f'{tmp}/e{i}.sock')
+ lo, med, hi = min(got), statistics.median(got), max(got)
+ added = med - base
+ elapsed = transfer(f'{tmp}/s{i}.sock', size)
+ mbs = size / (elapsed - 2 * delay / 1000) / 1e6
+ # The RTT is two delays (scheduling adds well under a millisecond),
+ # and the delay costs a transfer its two delays, never a delay a
+ # chunk: serialised, 256 MiB in 256 KiB chunks would take 1024 delays.
+ good = 2 * delay - 0.05 <= lo and added <= 2 * delay + 1.5 and mbs >= min(200, raw / 4)
+ ok &= good
+ print(f'delay {delay} ms each way: rtt min/median/max {lo:.2f}/{med:.2f}/{hi:.2f} ms '
+ f'(adds {added:.2f}, want {2 * delay}), throughput {mbs:.0f} MB/s '
+ f'({elapsed:.3f} s for {size >> 20} MiB) {"ok" if good else "FAIL"}')
+ for listener in relays:
+ listener.close()
+ # A capped link: the rate holds, and the delay comes on top.
+ cap, delay, size = 20.0, 40, 40 << 20
+ at = f'{tmp}/cap.sock'
+ listener = listen(at)
+ threading.Thread(target=serve, args=(listener, sink, delay, cap), daemon=True).start()
+ elapsed = transfer(at, size)
+ mbs = size / (elapsed - 2 * delay / 1000) / 1e6
+ good = abs(mbs - cap) / cap < 0.05
+ ok &= good
+ print(f'cap {cap} MB/s, delay {delay} ms: {mbs:.2f} MB/s ({elapsed:.3f} s for {size >> 20} MiB) '
+ f'{"ok" if good else "FAIL"}')
+ listener.close()
+ for f in os.listdir(tmp):
+ os.unlink(f'{tmp}/{f}')
+ os.rmdir(tmp)
+ print('self-check', 'ok' if ok else 'FAILED')
+ return 0 if ok else 1
+
+
+def main():
+ if '--self-check' in sys.argv:
+ sys.exit(self_check())
+ ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
+ ap.add_argument('listen')
+ ap.add_argument('target')
+ ap.add_argument('--delay', type=float, required=True, help='one-way delay, ms')
+ ap.add_argument('--rate', type=float, help='a bandwidth cap, MB/s')
+ a = ap.parse_args()
+ listener = listen(a.listen)
+ print(f'relaying {a.listen} -> {a.target}, {a.delay} ms each way'
+ + (f', {a.rate} MB/s' if a.rate else ''), flush=True)
+ try:
+ serve(listener, a.target, a.delay, a.rate)
+ except KeyboardInterrupt:
+ pass
+
+
+if __name__ == '__main__':
+ main()