1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
|
#!/usr/bin/env python3
"""A link with latency, for the replica gates (docs/next-daemon.md, phase 5):
it accepts on one socket, connects each client on to another, and holds
every chunk `delay` ms each way, timed from its arrival, so a stream's
throughput is not serialised behind the delay and its bytes stay in order.
An optional rate cap makes the link a bottleneck too: a chunk leaves once
the ones before it have crossed at that rate, then waits out the delay.
python3 -B test/delay_relay.py LISTEN TARGET --delay MS [--rate MB/s]
python3 -B test/delay_relay.py --self-check
An address is a path (a Unix socket) or host:port (TCP). The one-way
delays the gates use are 15, 40 and 103.5 ms: RTTs of 30, 80 and 207 ms
(the FreeBSD box's). The self-check measures the RTT and throughput it
adds at each, and with a cap.
"""
import argparse
import os
import queue
import socket
import statistics
import sys
import tempfile
import threading
import time
CHUNK = 256 << 10
def parse(address):
host, sep, port = address.rpartition(':')
if sep and port.isdigit() and '/' not in address:
return socket.AF_INET6 if ':' in host else socket.AF_INET, (host or '127.0.0.1', int(port))
return socket.AF_UNIX, address
def dial(address):
family, addr = parse(address)
s = socket.socket(family)
s.connect(addr)
if family != socket.AF_UNIX:
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
return s
def listen(address):
family, addr = parse(address)
s = socket.socket(family)
if family == socket.AF_UNIX:
try:
os.unlink(addr)
except FileNotFoundError:
pass
else:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(addr)
s.listen(64)
return s
def pump(src, dst, delay, rate, done):
"""src -> dst, each chunk due `delay` s after it arrived (and after the
ones before it crossed at `rate` bytes/s), in order; its end a shutdown."""
due_q = queue.SimpleQueue()
def reader():
free = 0.0 # when the capped link has sent what it holds
while True:
try:
data = src.recv(CHUNK)
except OSError:
data = b''
now = time.monotonic()
if rate:
free = max(free, now) + len(data) / rate
now = free
due_q.put((now + delay, data))
if not data:
return
def writer():
try:
while True:
due, data = due_q.get()
wait = due - time.monotonic()
if wait > 0:
time.sleep(wait)
if not data:
dst.shutdown(socket.SHUT_WR)
return
dst.sendall(data)
except OSError:
pass
finally:
done()
threading.Thread(target=reader, daemon=True).start()
threading.Thread(target=writer, daemon=True).start()
def serve(listener, target, delay_ms, rate_mbs=None):
"""Relays every connection `listener` accepts until it is closed."""
delay = delay_ms / 1000
rate = rate_mbs * 1e6 if rate_mbs else None
while True:
try:
client, _ = listener.accept()
except OSError:
return
try:
upstream = dial(target)
except OSError:
client.close()
continue
if client.family != socket.AF_UNIX:
client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
left = [2]
lock = threading.Lock()
def done(a=client, b=upstream, left=left, lock=lock):
with lock:
left[0] -= 1
if left[0] == 0:
a.close()
b.close()
pump(client, upstream, delay, rate, done)
pump(upstream, client, delay, rate, done)
# ---- the self-check ----
def echo_server(listener):
def one(c):
with c:
while data := c.recv(CHUNK):
c.sendall(data)
def loop():
while True:
try:
c, _ = listener.accept()
except OSError:
return
threading.Thread(target=one, args=(c,), daemon=True).start()
threading.Thread(target=loop, daemon=True).start()
def sink_server(listener):
"""Reads to the end, then answers one byte: a transfer's end to end."""
def one(c):
with c:
while c.recv(CHUNK):
pass
c.sendall(b'.')
def loop():
while True:
try:
c, _ = listener.accept()
except OSError:
return
threading.Thread(target=one, args=(c,), daemon=True).start()
threading.Thread(target=loop, daemon=True).start()
def rtts(address, n=40):
s = dial(address)
out = []
with s:
for i in range(n):
t = time.monotonic()
s.sendall(i.to_bytes(8, 'little'))
got = b''
while len(got) < 8:
got += s.recv(8 - len(got))
out.append((time.monotonic() - t) * 1000)
return out
def transfer(address, size):
"""Seconds from the first byte sent to the sink's answer."""
s = dial(address)
block = b'x' * CHUNK
with s:
t = time.monotonic()
left = size
while left > 0:
s.sendall(block[:min(left, CHUNK)])
left -= CHUNK
s.shutdown(socket.SHUT_WR)
assert s.recv(1) == b'.'
return time.monotonic() - t
def self_check():
tmp = tempfile.mkdtemp(prefix='relay-', dir=os.environ.get('TMPDIR'))
echo, sink = f'{tmp}/echo.sock', f'{tmp}/sink.sock'
echo_server(listen(echo))
sink_server(listen(sink))
ok = True
base = statistics.median(rtts(echo))
size = 256 << 20
raw = size / transfer(sink, size) / 1e6
print(f'direct: rtt median {base:.3f} ms, throughput {raw:.0f} MB/s')
for i, delay in enumerate([15, 40, 103.5]):
relays = []
for name, target in (('e', echo), ('s', sink)):
at = f'{tmp}/{name}{i}.sock'
listener = listen(at)
relays.append(listener)
threading.Thread(target=serve, args=(listener, target, delay), daemon=True).start()
got = rtts(f'{tmp}/e{i}.sock')
lo, med, hi = min(got), statistics.median(got), max(got)
added = med - base
elapsed = transfer(f'{tmp}/s{i}.sock', size)
mbs = size / (elapsed - 2 * delay / 1000) / 1e6
# The RTT is two delays (scheduling adds well under a millisecond),
# and the delay costs a transfer its two delays, never a delay a
# chunk: serialised, 256 MiB in 256 KiB chunks would take 1024 delays.
good = 2 * delay - 0.05 <= lo and added <= 2 * delay + 1.5 and mbs >= min(200, raw / 4)
ok &= good
print(f'delay {delay} ms each way: rtt min/median/max {lo:.2f}/{med:.2f}/{hi:.2f} ms '
f'(adds {added:.2f}, want {2 * delay}), throughput {mbs:.0f} MB/s '
f'({elapsed:.3f} s for {size >> 20} MiB) {"ok" if good else "FAIL"}')
for listener in relays:
listener.close()
# A capped link: the rate holds, and the delay comes on top.
cap, delay, size = 20.0, 40, 40 << 20
at = f'{tmp}/cap.sock'
listener = listen(at)
threading.Thread(target=serve, args=(listener, sink, delay, cap), daemon=True).start()
elapsed = transfer(at, size)
mbs = size / (elapsed - 2 * delay / 1000) / 1e6
good = abs(mbs - cap) / cap < 0.05
ok &= good
print(f'cap {cap} MB/s, delay {delay} ms: {mbs:.2f} MB/s ({elapsed:.3f} s for {size >> 20} MiB) '
f'{"ok" if good else "FAIL"}')
listener.close()
for f in os.listdir(tmp):
os.unlink(f'{tmp}/{f}')
os.rmdir(tmp)
print('self-check', 'ok' if ok else 'FAILED')
return 0 if ok else 1
def main():
if '--self-check' in sys.argv:
sys.exit(self_check())
ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0])
ap.add_argument('listen')
ap.add_argument('target')
ap.add_argument('--delay', type=float, required=True, help='one-way delay, ms')
ap.add_argument('--rate', type=float, help='a bandwidth cap, MB/s')
a = ap.parse_args()
listener = listen(a.listen)
print(f'relaying {a.listen} -> {a.target}, {a.delay} ms each way'
+ (f', {a.rate} MB/s' if a.rate else ''), flush=True)
try:
serve(listener, a.target, a.delay, a.rate)
except KeyboardInterrupt:
pass
if __name__ == '__main__':
main()
|