summaryrefslogtreecommitdiff
path: root/test/delay_relay.py
blob: c600a3aa2bea7b56191dadd426c82617e79d365d (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
#!/usr/bin/env python3
"""A link with latency, for the replica gates (docs/next-daemon.md, phase 5):
it accepts on one socket, connects each client on to another, and holds
every chunk `delay` ms each way, timed from its arrival, so a stream's
throughput is not serialised behind the delay and its bytes stay in order.
An optional rate cap makes the link a bottleneck too: a chunk leaves once
the ones before it have crossed at that rate, then waits out the delay. An
optional jitter adds up to that many ms more to each chunk's delay, at
random, its order kept (a chunk never overtakes the one before it).

    python3 -B test/delay_relay.py LISTEN TARGET --delay MS [--rate MB/s] [--jitter MS]
    python3 -B test/delay_relay.py --self-check

An address is a path (a Unix socket) or host:port (TCP). The one-way
delays the gates use are 15, 40 and 103.5 ms: RTTs of 30, 80 and 207 ms
(the FreeBSD box's). The self-check measures the RTT and throughput it
adds at each, and with a cap.
"""
import argparse
import os
import queue
import random
import socket
import statistics
import sys
import tempfile
import threading
import time

CHUNK = 256 << 10


def parse(address):
    host, sep, port = address.rpartition(':')
    if sep and port.isdigit() and '/' not in address:
        return socket.AF_INET6 if ':' in host else socket.AF_INET, (host or '127.0.0.1', int(port))
    return socket.AF_UNIX, address


def dial(address):
    family, addr = parse(address)
    s = socket.socket(family)
    s.connect(addr)
    if family != socket.AF_UNIX:
        s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
    return s


def listen(address):
    family, addr = parse(address)
    s = socket.socket(family)
    if family == socket.AF_UNIX:
        try:
            os.unlink(addr)
        except FileNotFoundError:
            pass
    else:
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    s.bind(addr)
    s.listen(64)
    return s


def pump(src, dst, delay, rate, done, jitter=0.0):
    """src -> dst, each chunk due `delay` s after it arrived (and after the
    ones before it crossed at `rate` bytes/s), plus up to `jitter` s at
    random but never before the chunk ahead of it; its end a shutdown."""
    due_q = queue.SimpleQueue()

    def reader():
        free = 0.0  # when the capped link has sent what it holds
        last = 0.0  # when the chunk before is due: order is kept
        while True:
            try:
                data = src.recv(CHUNK)
            except OSError:
                data = b''
            now = time.monotonic()
            if rate:
                free = max(free, now) + len(data) / rate
                now = free
            last = max(last, now + delay + (random.uniform(0, jitter) if jitter else 0))
            due_q.put((last, data))
            if not data:
                return

    def writer():
        try:
            while True:
                due, data = due_q.get()
                wait = due - time.monotonic()
                if wait > 0:
                    time.sleep(wait)
                if not data:
                    dst.shutdown(socket.SHUT_WR)
                    return
                dst.sendall(data)
        except OSError:
            pass
        finally:
            done()

    threading.Thread(target=reader, daemon=True).start()
    threading.Thread(target=writer, daemon=True).start()


def serve(listener, target, delay_ms, rate_mbs=None, jitter_ms=0):
    """Relays every connection `listener` accepts until it is closed."""
    delay = delay_ms / 1000
    jitter = jitter_ms / 1000
    rate = rate_mbs * 1e6 if rate_mbs else None
    while True:
        try:
            client, _ = listener.accept()
        except OSError:
            return
        try:
            upstream = dial(target)
        except OSError:
            client.close()
            continue
        if client.family != socket.AF_UNIX:
            client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        left = [2]
        lock = threading.Lock()

        def done(a=client, b=upstream, left=left, lock=lock):
            with lock:
                left[0] -= 1
                if left[0] == 0:
                    a.close()
                    b.close()

        pump(client, upstream, delay, rate, done, jitter)
        pump(upstream, client, delay, rate, done, jitter)


# ---- the self-check ----

def echo_server(listener):
    def one(c):
        with c:
            while data := c.recv(CHUNK):
                c.sendall(data)

    def loop():
        while True:
            try:
                c, _ = listener.accept()
            except OSError:
                return
            threading.Thread(target=one, args=(c,), daemon=True).start()
    threading.Thread(target=loop, daemon=True).start()


def sink_server(listener):
    """Reads to the end, then answers one byte: a transfer's end to end."""
    def one(c):
        with c:
            while c.recv(CHUNK):
                pass
            c.sendall(b'.')

    def loop():
        while True:
            try:
                c, _ = listener.accept()
            except OSError:
                return
            threading.Thread(target=one, args=(c,), daemon=True).start()
    threading.Thread(target=loop, daemon=True).start()


def rtts(address, n=40):
    s = dial(address)
    out = []
    with s:
        for i in range(n):
            t = time.monotonic()
            s.sendall(i.to_bytes(8, 'little'))
            got = b''
            while len(got) < 8:
                got += s.recv(8 - len(got))
            out.append((time.monotonic() - t) * 1000)
    return out


def transfer(address, size):
    """Seconds from the first byte sent to the sink's answer."""
    s = dial(address)
    block = b'x' * CHUNK
    with s:
        t = time.monotonic()
        left = size
        while left > 0:
            s.sendall(block[:min(left, CHUNK)])
            left -= CHUNK
        s.shutdown(socket.SHUT_WR)
        assert s.recv(1) == b'.'
        return time.monotonic() - t


def self_check():
    tmp = tempfile.mkdtemp(prefix='relay-', dir=os.environ.get('TMPDIR'))
    echo, sink = f'{tmp}/echo.sock', f'{tmp}/sink.sock'
    echo_server(listen(echo))
    sink_server(listen(sink))
    ok = True
    base = statistics.median(rtts(echo))
    size = 256 << 20
    raw = size / transfer(sink, size) / 1e6
    print(f'direct: rtt median {base:.3f} ms, throughput {raw:.0f} MB/s')
    for i, delay in enumerate([15, 40, 103.5]):
        relays = []
        for name, target in (('e', echo), ('s', sink)):
            at = f'{tmp}/{name}{i}.sock'
            listener = listen(at)
            relays.append(listener)
            threading.Thread(target=serve, args=(listener, target, delay), daemon=True).start()
        got = rtts(f'{tmp}/e{i}.sock')
        lo, med, hi = min(got), statistics.median(got), max(got)
        added = med - base
        elapsed = transfer(f'{tmp}/s{i}.sock', size)
        mbs = size / (elapsed - 2 * delay / 1000) / 1e6
        # The RTT is two delays (scheduling adds well under a millisecond),
        # and the delay costs a transfer its two delays, never a delay a
        # chunk: serialised, 256 MiB in 256 KiB chunks would take 1024 delays.
        good = 2 * delay - 0.05 <= lo and added <= 2 * delay + 1.5 and mbs >= min(200, raw / 4)
        ok &= good
        print(f'delay {delay} ms each way: rtt min/median/max {lo:.2f}/{med:.2f}/{hi:.2f} ms '
              f'(adds {added:.2f}, want {2 * delay}), throughput {mbs:.0f} MB/s '
              f'({elapsed:.3f} s for {size >> 20} MiB) {"ok" if good else "FAIL"}')
        for listener in relays:
            listener.close()
    # A capped link: the rate holds, and the delay comes on top.
    cap, delay, size = 20.0, 40, 40 << 20
    at = f'{tmp}/cap.sock'
    listener = listen(at)
    threading.Thread(target=serve, args=(listener, sink, delay, cap), daemon=True).start()
    elapsed = transfer(at, size)
    mbs = size / (elapsed - 2 * delay / 1000) / 1e6
    good = abs(mbs - cap) / cap < 0.05
    ok &= good
    print(f'cap {cap} MB/s, delay {delay} ms: {mbs:.2f} MB/s ({elapsed:.3f} s for {size >> 20} MiB) '
          f'{"ok" if good else "FAIL"}')
    listener.close()
    for f in os.listdir(tmp):
        os.unlink(f'{tmp}/{f}')
    os.rmdir(tmp)
    print('self-check', 'ok' if ok else 'FAILED')
    return 0 if ok else 1


def main():
    if '--self-check' in sys.argv:
        sys.exit(self_check())
    ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
    ap.add_argument('listen')
    ap.add_argument('target')
    ap.add_argument('--delay', type=float, required=True, help='one-way delay, ms')
    ap.add_argument('--rate', type=float, help='a bandwidth cap, MB/s')
    ap.add_argument('--jitter', type=float, default=0, help='up to this many ms more a chunk, at random')
    a = ap.parse_args()
    listener = listen(a.listen)
    print(f'relaying {a.listen} -> {a.target}, {a.delay} ms each way'
          + (f', {a.rate} MB/s' if a.rate else '') + (f', jitter {a.jitter} ms' if a.jitter else ''), flush=True)
    try:
        serve(listener, a.target, a.delay, a.rate, a.jitter)
    except KeyboardInterrupt:
        pass


if __name__ == '__main__':
    main()