const std = @import("std"); const libc = std.c; const ssl = @import("openssl"); comptime { if (ssl.OPENSSL_VERSION_NUMBER < 0x30600000) @compileError("9P over QUIC requires OpenSSL 3.6 or newer"); } pub const alpn = "pardes-9p"; pub const Error = error{ Tls, Socket, SocketFlags, SocketOption, Bind, Address, Closed, InvalidWrite }; pub const Listener = struct { fd: c_int, handle: *ssl.SSL, address: std.Io.net.IpAddress, pub fn init(address: std.Io.net.IpAddress) Error!Listener { ssl.ERR_clear_error(); const ctx = ssl.SSL_CTX_new(ssl.OSSL_QUIC_server_method()) orelse return error.Tls; defer ssl.SSL_CTX_free(ctx); const key = ssl.EVP_PKEY_Q_keygen(null, null, "EC", @as([*:0]const u8, "prime256v1")) orelse return error.Tls; defer ssl.EVP_PKEY_free(key); const cert = ssl.X509_new() orelse return error.Tls; defer ssl.X509_free(cert); if (ssl.X509_set_version(cert, 2) != 1 or ssl.ASN1_INTEGER_set(ssl.X509_get_serialNumber(cert), 1) != 1 or ssl.X509_gmtime_adj(ssl.X509_getm_notBefore(cert), -60) == null or ssl.X509_gmtime_adj(ssl.X509_getm_notAfter(cert), 365 * 24 * 60 * 60) == null or ssl.X509_set_pubkey(cert, key) != 1) return error.Tls; const name = ssl.X509_get_subject_name(cert) orelse return error.Tls; if (ssl.X509_NAME_add_entry_by_txt(name, "CN", ssl.MBSTRING_ASC, "pardes", -1, -1, 0) != 1 or ssl.X509_set_issuer_name(cert, name) != 1 or ssl.X509_sign(cert, key, ssl.EVP_sha256()) <= 0 or ssl.SSL_CTX_use_certificate(ctx, cert) != 1 or ssl.SSL_CTX_use_PrivateKey(ctx, key) != 1) return error.Tls; ssl.SSL_CTX_set_verify(ctx, ssl.SSL_VERIFY_NONE, null); ssl.SSL_CTX_set_alpn_select_cb(ctx, selectAlpn, null); var addr: libc.sockaddr.storage = undefined; const addr_len = sockaddr(address, &addr); const fd = try udp(addr.family); errdefer _ = libc.close(fd); if (libc.bind(fd, @ptrCast(&addr), addr_len) != 0) return error.Bind; var actual_len: libc.socklen_t = @sizeOf(@TypeOf(addr)); if (libc.getsockname(fd, @ptrCast(&addr), &actual_len) != 0) return error.Address; var actual = address; actual.setPort(switch (address) { .ip4 => std.mem.bigToNative(u16, @as(*const libc.sockaddr.in, @ptrCast(&addr)).port), .ip6 => std.mem.bigToNative(u16, @as(*const libc.sockaddr.in6, @ptrCast(&addr)).port), }); const handle = ssl.SSL_new_listener(ctx, 0) orelse return error.Tls; errdefer ssl.SSL_free(handle); if (ssl.SSL_set_fd(handle, fd) != 1 or ssl.SSL_set_blocking_mode(handle, 0) != 1 or ssl.SSL_listen(handle) != 1) return error.Tls; return .{ .fd = fd, .handle = handle, .address = actual }; } pub fn accept(l: *Listener) Error!?Connection { ssl.ERR_clear_error(); const handle = ssl.SSL_accept_connection(l.handle, ssl.SSL_ACCEPT_CONNECTION_NO_BLOCK) orelse { if (ssl.ERR_peek_error() != 0) return error.Tls; return null; }; errdefer ssl.SSL_free(handle); if (ssl.SSL_set_default_stream_mode(handle, ssl.SSL_DEFAULT_STREAM_MODE_NONE) != 1 or ssl.SSL_set_blocking_mode(handle, 0) != 1) return error.Tls; return .{ .handle = handle }; } pub fn events(l: *Listener) Error!void { ssl.ERR_clear_error(); if (ssl.SSL_handle_events(l.handle) != 1) return error.Tls; } pub fn poll(l: *const Listener) libc.pollfd { return pollFd(l.handle, l.fd); } pub fn nextDue(l: *const Listener) ?i32 { return due(l.handle); } // Accepted connections must be released before the shared UDP socket. pub fn deinit(l: *Listener) void { ssl.SSL_free(l.handle); _ = libc.close(l.fd); l.* = undefined; } }; pub const Connection = struct { handle: *ssl.SSL, stream: ?*ssl.SSL = null, fd: c_int = -1, pending_write_len: usize = 0, pub fn dial(address: std.Io.net.IpAddress) Error!Connection { ssl.ERR_clear_error(); const ctx = ssl.SSL_CTX_new(ssl.OSSL_QUIC_client_method()) orelse return error.Tls; defer ssl.SSL_CTX_free(ctx); ssl.SSL_CTX_set_verify(ctx, ssl.SSL_VERIFY_NONE, null); const fd = try udp(if (address == .ip4) libc.AF.INET else libc.AF.INET6); errdefer _ = libc.close(fd); const handle = ssl.SSL_new(ctx) orelse return error.Tls; errdefer ssl.SSL_free(handle); if (ssl.SSL_set_fd(handle, fd) != 1 or ssl.SSL_set_blocking_mode(handle, 0) != 1 or ssl.SSL_set_default_stream_mode(handle, ssl.SSL_DEFAULT_STREAM_MODE_NONE) != 1) return error.Tls; const protocols = [_]u8{alpn.len} ++ alpn.*; if (ssl.SSL_set_alpn_protos(handle, &protocols, protocols.len) != 0) return error.Tls; const peer = ssl.BIO_ADDR_new() orelse return error.Tls; defer ssl.BIO_ADDR_free(peer); const made = switch (address) { .ip4 => |ip| ssl.BIO_ADDR_rawmake(peer, libc.AF.INET, &ip.bytes, ip.bytes.len, std.mem.nativeToBig(u16, ip.port)), .ip6 => |ip| ssl.BIO_ADDR_rawmake(peer, libc.AF.INET6, &ip.bytes, ip.bytes.len, std.mem.nativeToBig(u16, ip.port)), }; if (made != 1 or ssl.SSL_set1_initial_peer_addr(handle, peer) != 1) return error.Tls; return .{ .handle = handle, .fd = fd }; } pub fn handshake(c: *Connection) Error!bool { ssl.ERR_clear_error(); var close_info: ssl.SSL_CONN_CLOSE_INFO = undefined; if (ssl.SSL_get_conn_close_info(c.handle, &close_info, @sizeOf(@TypeOf(close_info))) == 1) return error.Closed; if (ssl.SSL_is_init_finished(c.handle) == 1) return true; const rc = if (c.fd >= 0) ssl.SSL_connect(c.handle) else ssl.SSL_accept(c.handle); if (rc == 1) return true; try retry(c.handle, rc); return false; } fn ready(c: *Connection) Error!bool { if (!try c.handshake()) return false; if (c.stream != null) return true; ssl.ERR_clear_error(); const stream = if (c.fd >= 0) ssl.SSL_new_stream(c.handle, ssl.SSL_STREAM_FLAG_NO_BLOCK) else ssl.SSL_accept_stream(c.handle, ssl.SSL_ACCEPT_STREAM_NO_BLOCK); if (stream == null) { if (ssl.ERR_peek_error() != 0) return error.Tls; return false; } errdefer ssl.SSL_free(stream); if (ssl.SSL_set_blocking_mode(stream, 0) != 1 or ssl.SSL_get_stream_id(stream) != 0 or ssl.SSL_set_incoming_stream_policy(c.handle, ssl.SSL_INCOMING_STREAM_POLICY_REJECT, 0) != 1) return error.Tls; _ = ssl.SSL_set_mode(stream, ssl.SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); c.stream = stream; return true; } pub fn read(c: *Connection, bytes: []u8) Error!?usize { if (!try c.ready()) return null; ssl.ERR_clear_error(); var len: usize = 0; const rc = ssl.SSL_read_ex(c.stream, bytes.ptr, bytes.len, &len); if (rc == 1) return len; if (ssl.SSL_get_error(c.stream, rc) == ssl.SSL_ERROR_ZERO_RETURN) return 0; try retry(c.stream.?, rc); return null; } pub fn pending(c: *const Connection) bool { var close_info: ssl.SSL_CONN_CLOSE_INFO = undefined; if (ssl.SSL_get_conn_close_info(c.handle, &close_info, @sizeOf(@TypeOf(close_info))) == 1) return true; if (c.stream) |stream| { var item: ssl.SSL_POLL_ITEM = .{ .desc = ssl.SSL_as_poll_descriptor(stream), .events = ssl.SSL_POLL_EVENT_RE, .revents = 0 }; const timeout: ssl.struct_timeval = .{ .tv_sec = 0, .tv_usec = 0 }; if (ssl.SSL_poll(&item, 1, @sizeOf(@TypeOf(item)), &timeout, ssl.SSL_POLL_FLAG_NO_HANDLE_EVENTS, null) != 1) return true; return item.revents != 0; } return ssl.SSL_get_accept_stream_queue_len(c.handle) != 0; } pub fn write(c: *Connection, bytes: []const u8) Error!usize { if (!try c.ready()) return 0; if (bytes.len < c.pending_write_len) return error.InvalidWrite; const requested = if (c.pending_write_len != 0) c.pending_write_len else bytes.len; if (requested == 0) return 0; ssl.ERR_clear_error(); var len: usize = 0; const rc = ssl.SSL_write_ex(c.stream, bytes.ptr, requested, &len); if (rc == 1) { c.pending_write_len = 0; return len; } try retry(c.stream.?, rc); c.pending_write_len = requested; return 0; } pub fn conclude(c: *Connection) Error!void { if (c.pending_write_len != 0) return error.InvalidWrite; if (!try c.ready()) return error.Closed; ssl.ERR_clear_error(); if (ssl.SSL_stream_conclude(c.stream, 0) != 1) return error.Tls; } pub fn events(c: *Connection) Error!void { if (c.fd < 0) return; ssl.ERR_clear_error(); if (ssl.SSL_handle_events(c.handle) != 1) return error.Tls; } pub fn poll(c: *const Connection) ?libc.pollfd { return if (c.fd >= 0) pollFd(c.handle, c.fd) else null; } pub fn nextDue(c: *const Connection) ?i32 { return if (c.fd >= 0) due(c.handle) else null; } pub fn deinit(c: *Connection) void { ssl.ERR_clear_error(); _ = ssl.SSL_shutdown_ex(c.handle, ssl.SSL_SHUTDOWN_FLAG_RAPID | ssl.SSL_SHUTDOWN_FLAG_NO_STREAM_FLUSH | ssl.SSL_SHUTDOWN_FLAG_NO_BLOCK, null, 0); ssl.SSL_free(c.stream); ssl.SSL_free(c.handle); if (c.fd >= 0) _ = libc.close(c.fd); c.* = undefined; } }; fn udp(family: u16) Error!c_int { const fd = libc.socket(family, libc.SOCK.DGRAM, 0); if (fd < 0) return error.Socket; errdefer _ = libc.close(fd); const flags = libc.fcntl(fd, libc.F.GETFL, @as(c_int, 0)); if (flags < 0) return error.SocketFlags; var options: libc.O = @bitCast(@as(u32, @bitCast(flags))); options.NONBLOCK = true; if (libc.fcntl(fd, libc.F.SETFL, @as(c_int, @bitCast(@as(u32, @bitCast(options))))) != 0 or libc.fcntl(fd, libc.F.SETFD, @as(c_int, libc.FD_CLOEXEC)) != 0) return error.SocketFlags; if (family == libc.AF.INET6) { const enabled: c_int = 1; const v6only = if (@import("builtin").os.tag.isDarwin()) 27 else libc.IPV6.V6ONLY; if (libc.setsockopt(fd, libc.IPPROTO.IPV6, v6only, &enabled, @sizeOf(c_int)) != 0) return error.SocketOption; } return fd; } fn sockaddr(address: std.Io.net.IpAddress, out: *libc.sockaddr.storage) libc.socklen_t { switch (address) { .ip4 => |ip| { const addr: *libc.sockaddr.in = @ptrCast(out); addr.* = .{ .port = std.mem.nativeToBig(u16, ip.port), .addr = @bitCast(ip.bytes) }; return @sizeOf(libc.sockaddr.in); }, .ip6 => |ip| { const addr: *libc.sockaddr.in6 = @ptrCast(out); addr.* = .{ .port = std.mem.nativeToBig(u16, ip.port), .addr = ip.bytes, .flowinfo = 0, .scope_id = 0 }; return @sizeOf(libc.sockaddr.in6); }, } } fn pollFd(handle: *ssl.SSL, fd: c_int) libc.pollfd { var result: libc.pollfd = .{ .fd = fd, .events = 0, .revents = 0 }; if (ssl.SSL_net_read_desired(handle) == 1) result.events |= libc.POLL.IN; if (ssl.SSL_net_write_desired(handle) == 1) result.events |= libc.POLL.OUT; return result; } fn due(handle: *ssl.SSL) ?i32 { var tv: ssl.struct_timeval = undefined; var infinite: c_int = undefined; if (ssl.SSL_get_event_timeout(handle, &tv, &infinite) != 1) return 0; if (infinite != 0) return null; const ms = @as(i128, tv.tv_sec) * 1000 + @divFloor(@as(i128, tv.tv_usec) + 999, 1000); return @intCast(std.math.clamp(ms, 0, std.math.maxInt(i32))); } fn retry(handle: *ssl.SSL, rc: c_int) Error!void { switch (ssl.SSL_get_error(handle, rc)) { ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE => {}, ssl.SSL_ERROR_ZERO_RETURN => return error.Closed, else => return error.Tls, } } fn selectAlpn(_: ?*ssl.SSL, out: [*c][*c]const u8, outlen: [*c]u8, input: [*c]const u8, len: c_uint, _: ?*anyopaque) callconv(.c) c_int { var offset: usize = 0; while (offset < len) { const size = input[offset]; offset += 1; if (size > len - offset) return ssl.SSL_TLSEXT_ERR_ALERT_FATAL; if (std.mem.eql(u8, input[offset..][0..size], alpn)) { out.* = input + offset; outlen.* = size; return ssl.SSL_TLSEXT_ERR_OK; } offset += size; } return ssl.SSL_TLSEXT_ERR_ALERT_FATAL; } const TestPair = struct { listener: *Listener, client: Connection, server: ?Connection = null, fn init(listener: *Listener) !TestPair { var p: TestPair = .{ .listener = listener, .client = try .dial(listener.address) }; errdefer p.deinit(); const deadline = testNow() + 3000; while (true) { try listener.events(); try p.client.events(); if (p.server == null) p.server = try listener.accept(); const connected = try p.client.handshake(); if (p.server) |*server| if (connected and try server.handshake()) return p; try p.wait(deadline); } } fn wait(p: *TestPair, deadline: i64) !void { const remaining = deadline - testNow(); if (remaining <= 0) return error.Deadline; var timeout: i32 = @intCast(@min(remaining, std.math.maxInt(i32))); if (p.listener.nextDue()) |ms| timeout = @min(timeout, ms); if (p.client.nextDue()) |ms| timeout = @min(timeout, ms); var fds = [_]libc.pollfd{ p.listener.poll(), p.client.poll().? }; const rc = libc.poll(&fds, fds.len, timeout); if (rc < 0 and libc.errno(rc) != .INTR) return error.Poll; if (testNow() >= deadline) return error.Deadline; try p.listener.events(); try p.client.events(); } fn transfer(p: *TestPair, from_client: bool, bytes: []const u8, fragment: usize) !void { const writer = if (from_client) &p.client else &p.server.?; const reader = if (from_client) &p.server.? else &p.client; var sent: usize = 0; var received: usize = 0; var buffer: [8192]u8 = undefined; const deadline = testNow() + 3000; while (received < bytes.len) { const written = if (sent != bytes.len) try writer.write(bytes[sent..][0..@min(fragment, bytes.len - sent)]) else 0; sent += written; const count = try reader.read(buffer[0..@min(fragment, buffer.len)]); if (count) |n| { try std.testing.expect(n > 0 and n <= bytes.len - received); try std.testing.expectEqualSlices(u8, bytes[received..][0..n], buffer[0..n]); received += n; } if (testNow() >= deadline) return error.Deadline; if (received != bytes.len and written == 0 and count == null) { try p.wait(deadline); } else { try p.listener.events(); try p.client.events(); } } try std.testing.expectEqual(bytes.len, sent); } fn finish(p: *TestPair) !void { try p.client.conclude(); try p.server.?.conclude(); var buffer: [16]u8 = undefined; var a = false; var b = false; const deadline = testNow() + 3000; while (!a or !b) { if (!a) if (try p.client.read(&buffer)) |n| { try std.testing.expectEqual(@as(usize, 0), n); a = true; }; if (!b) if (try p.server.?.read(&buffer)) |n| { try std.testing.expectEqual(@as(usize, 0), n); b = true; }; if (!a or !b) try p.wait(deadline); } } fn deinit(p: *TestPair) void { if (p.server) |*server| server.deinit(); p.client.deinit(); } }; fn testNow() i64 { var ts: libc.timespec = undefined; std.debug.assert(libc.clock_gettime(.MONOTONIC, &ts) == 0); return @as(i64, @intCast(ts.sec)) * 1000 + @divFloor(@as(i64, @intCast(ts.nsec)), 1_000_000); } test "QUIC fragmented 9P frames reconnect and stream EOF over IPv4 and IPv6" { const request = "\x13\x00\x00\x00\x64\xff\xff\x00\x20\x00\x00\x06\x00" ++ "9P2000"; const response = "\x13\x00\x00\x00\x65\xff\xff\x00\x20\x00\x00\x06\x00" ++ "9P2000"; const read_request = "\x17\x00\x00\x00\x74\x01\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00"; const read_response = "\x16\x00\x00\x00\x75\x01\x00\x0b\x00\x00\x00" ++ "hello ninep"; for ([_][]const u8{ "127.0.0.1", "::1" }) |host| { var listener = try Listener.init(try .parse(host, 0)); defer listener.deinit(); try std.testing.expect(listener.address.getPort() != 0); if (listener.address == .ip6) { var enabled: c_int = 0; var len: libc.socklen_t = @sizeOf(c_int); const v6only = if (@import("builtin").os.tag.isDarwin()) 27 else libc.IPV6.V6ONLY; try std.testing.expectEqual(@as(c_int, 0), libc.getsockopt(listener.fd, libc.IPPROTO.IPV6, v6only, @ptrCast(&enabled), &len)); try std.testing.expectEqual(@as(c_int, 1), enabled); } for (0..3) |_| { var pair = try TestPair.init(&listener); defer pair.deinit(); try std.testing.expect(pair.server.?.poll() == null); try std.testing.expect(pair.server.?.nextDue() == null); for ([_]usize{ 1, 2, 7, 64 }) |fragment| { try pair.transfer(true, request, fragment); try pair.transfer(false, response, fragment); try pair.transfer(true, read_request, fragment); try pair.transfer(false, read_response, fragment); } try pair.finish(); } } } test "QUIC backpressure retries a moved prefix while new replies are appended" { var listener = try Listener.init(try .parse("127.0.0.1", 0)); defer listener.deinit(); var pair = try TestPair.init(&listener); defer pair.deinit(); try pair.transfer(true, "hello", 1); const bytes: [8192]u8 = @splat(0x5a); var total: usize = 0; const deadline = testNow() + 3000; while (total < 64 * 1024 * 1024) { const written = try pair.client.write(&bytes); total += written; if (written == 0) break; try pair.listener.events(); try pair.client.events(); if (testNow() >= deadline) return error.Deadline; } try std.testing.expect(total > 0 and total < 64 * 1024 * 1024); try std.testing.expectEqual(bytes.len, pair.client.pending_write_len); try std.testing.expectError(error.InvalidWrite, pair.client.write(bytes[0..1])); var buffer: [8192]u8 = undefined; var received: usize = 0; // A pending SSL write may already have delivered a prefix before it // reports completion. Leave that prefix for the retry check below. while (received < total) { if (try pair.server.?.read(buffer[0..@min(buffer.len, total - received)])) |n| { try std.testing.expect(n > 0); try std.testing.expect(std.mem.allEqual(u8, buffer[0..n], 0x5a)); received += n; } else try pair.wait(deadline); } try std.testing.expectEqual(total, received); var moved: [8192 + 7]u8 = undefined; @memcpy(moved[0..bytes.len], &bytes); @memset(moved[bytes.len..], 0x6b); while (true) { const n = try pair.client.write(&moved); if (n != 0) { try std.testing.expectEqual(bytes.len, n); break; } try pair.wait(deadline); } received = 0; while (received < bytes.len) { if (try pair.server.?.read(&buffer)) |n| { try std.testing.expect(n > 0); try std.testing.expect(std.mem.allEqual(u8, buffer[0..n], 0x5a)); received += n; } else try pair.wait(deadline); } try std.testing.expectEqual(bytes.len, received); try pair.transfer(true, moved[bytes.len..], 7); try pair.finish(); } test "QUIC owner can enforce handshake and read deadlines without busy polling" { var listener = try Listener.init(try .parse("127.0.0.1", 0)); defer listener.deinit(); var client = try Connection.dial(listener.address); defer client.deinit(); const handshake_deadline = testNow() + 100; var turns: usize = 0; while (testNow() < handshake_deadline) { try std.testing.expect(!try client.handshake()); try client.events(); var timeout: i32 = @intCast(@max(0, handshake_deadline - testNow())); if (client.nextDue()) |ms| timeout = @min(timeout, ms); var fds = [_]libc.pollfd{client.poll().?}; const rc = libc.poll(&fds, fds.len, timeout); if (rc < 0 and libc.errno(rc) != .INTR) return error.Poll; turns += 1; } try std.testing.expect(turns < 100); var serving = try Listener.init(try .parse("127.0.0.1", 0)); defer serving.deinit(); var pair = try TestPair.init(&serving); defer pair.deinit(); try pair.transfer(true, "request", 2); var buffer: [32]u8 = undefined; const read_deadline = testNow() + 100; turns = 0; while (true) { try std.testing.expect(try pair.client.read(&buffer) == null); pair.wait(read_deadline) catch |err| { try std.testing.expectEqual(error.Deadline, err); break; }; turns += 1; } try std.testing.expect(turns < 100); } test "QUIC bind failure leaves the existing listener usable" { var listener = try Listener.init(try .parse("127.0.0.1", 0)); defer listener.deinit(); for (0..8) |_| try std.testing.expectError(error.Bind, Listener.init(listener.address)); var pair = try TestPair.init(&listener); defer pair.deinit(); try pair.transfer(true, "still listening", 3); try pair.transfer(false, "still serving", 2); try pair.finish(); } test "QUIC pending reports buffered bytes and EOF without UDP readiness" { var listener = try Listener.init(try .parse("127.0.0.1", 0)); defer listener.deinit(); var pair = try TestPair.init(&listener); defer pair.deinit(); try std.testing.expect(!pair.server.?.pending()); try std.testing.expectEqual(@as(usize, 3), try pair.client.write("abc")); const deadline = testNow() + 3000; while (!pair.server.?.pending()) try pair.wait(deadline); var buffer: [3]u8 = undefined; try std.testing.expectEqual(@as(?usize, 1), try pair.server.?.read(buffer[0..1])); try std.testing.expectEqual(@as(u8, 'a'), buffer[0]); try listener.events(); try pair.client.events(); try std.testing.expect(pair.server.?.pending()); try std.testing.expectEqual(@as(?usize, 2), try pair.server.?.read(buffer[1..])); try std.testing.expectEqualStrings("abc", &buffer); try std.testing.expect(!pair.server.?.pending()); try pair.client.conclude(); while (!pair.server.?.pending()) try pair.wait(deadline); try std.testing.expectEqual(@as(?usize, 0), try pair.server.?.read(&buffer)); } test "QUIC moved listener retains its in-memory identity" { var original = try Listener.init(try .parse("127.0.0.1", 0)); var listener = original; original = undefined; defer listener.deinit(); var pair = try TestPair.init(&listener); defer pair.deinit(); try pair.transfer(true, "moved listener", 2); try pair.transfer(false, "same identity", 3); try pair.finish(); }