diff options
Diffstat (limited to '9ns/docs')
| -rw-r--r-- | 9ns/docs/DESIGN.md | 208 |
1 files changed, 152 insertions, 56 deletions
diff --git a/9ns/docs/DESIGN.md b/9ns/docs/DESIGN.md index 32c548c..ad59521 100644 --- a/9ns/docs/DESIGN.md +++ b/9ns/docs/DESIGN.md @@ -166,12 +166,13 @@ Threads and node ids: * A **mount** is a posted name walked into: the socket to dial, its own `nine.Session` and bridge state (inode table, handles — the ordinary single-server translation, unchanged) once dialed, and a **worker - thread** that owns all of it. The dispatcher makes a mount without + thread** that owns all of it (its transfer workers share the session). The dispatcher makes a mount without touching the network and hands it the walk; the worker dials while serving that first request. It pops copied requests off a queue and - serves them one at a time, exactly the single-connection contract; the - dispatcher keeps reading `/dev/fuse` meanwhile, so one slow server never - blocks the other names. Replies go straight back on the FUSE fd (one + serves them one at a time, exactly the single-connection contract + (READ and WRITE handed on to that mount's transfer workers, see + *Concurrency* under `src/bridge.zig`); the dispatcher keeps reading + `/dev/fuse` meanwhile, so one slow server never blocks the other names. Replies go straight back on the FUSE fd (one `writev` per reply; the kernel processes each write as one message). The queue is a plain list under a mutex and condition rather than an `std.Io.Queue` because an interrupt has to find and remove a request by @@ -353,15 +354,17 @@ pub fn nameAfter(comptime T: type, req: Request) ![]const u8; // NUL-terminated The request buffer must be ≥ `max_write + 4096`; 9ns uses 1 MiB + 4 KiB. Requests with an unknown/unsupported opcode get `ENOSYS`. -### `src/nine.zig` — synchronous 9P2000 session on a blocking fd +### `src/nine.zig` — 9P2000 session on a blocking fd, shared by threads -Thin, synchronous RPC layer over `cloud9.Client` (which is push/take, -non-blocking-agnostic). One outstanding request at a time (the FUSE loop is -single-threaded). Fids are allocated from a free list. +Thin RPC layer over `cloud9.Client` (which is push/take, +non-blocking-agnostic). Many requests may be in flight at once, each under +its own tag; replies come back in any order. The connection is a `Session`; +each thread that makes calls on it has a `Caller`. Fids are allocated from +a free list (under the session's mutex). ```zig pub const Address = union(enum) { unix: []const u8, tcp: struct { host: []const u8, port: u16 }, fd: i32 }; -/// Owner's hook into the reply wait (the bridge's FUSE fd); see "Interrupts" under bridge.zig. +/// Owner's hook into the reply wait (the bridge's FUSE fd, or an interrupt pipe); see "Interrupts" under bridge.zig. pub const Interrupt = struct { ctx: *anyopaque, watch: *const fn (ctx) i32, // fd to poll alongside the socket, or -1 @@ -370,53 +373,81 @@ pub const Interrupt = struct { }; pub const Session = struct { pub const Error = error{ Nine, Protocol, Io, Closed, Stopped, Interrupted, TooLarge, OutOfMemory }; - /// After error.Nine, `ename` holds the server's Rerror text (copied, bounded). - ename: [256]u8, ename_len: usize, msize: u32, - stop_fd: i32 = -1, // readable → pending rpc fails with error.Stopped - interrupt: ?Interrupt = null, + stop_fd: i32 = -1, // readable → every waiting call fails with error.Stopped + flush_grace_ms: i32 = 3000, // a Tflush unanswered this long wedges the session + wedged: bool, // declared gone: every call → error.Closed pub fn connect(gpa: std.mem.Allocator, address: Address, msize: u32) !Session; // socket+connect, version - pub fn deinit(s: *Session) void; - pub fn attach(s: *Session, fid: u32, uname: []const u8, aname: []const u8) Error!cloud9.Qid; + pub fn deinit(s: *Session) void; // after every Caller is done pub fn allocFid(s: *Session) u32; pub fn freeFid(s: *Session, fid: u32) void; - /// Generic RPC. Result slices borrow the input buffer until the next call. - pub fn rpc(s: *Session, req: cloud9.Client.Request) Error!cloud9.Client.Result; + pub fn over(s: *Session) bool; // wedged, or the connection died +}; +pub const Caller = struct { + session: *Session, + /// After error.Nine, `ename` holds the server's Rerror text (copied, bounded). + ename: [256]u8, ename_len: usize, + interrupt: ?Interrupt = null, + abort_on_cancel: bool = false, // handshake: a cancellation hangs up instead of flushing + + pub fn init(s: *Session) !Caller; // msize reply buffer + wake-up eventfd + pub fn deinit(c: *Caller) void; + /// Generic RPC. Result slices borrow the caller's reply buffer until its next call. + pub fn rpc(c: *Caller, req: cloud9.Client.Request) Error!cloud9.Client.Result; // Conveniences (all built on rpc): - pub fn walk(s, fid: u32, newfid: u32, names: []const []const u8) Error!Walk; // Walk = { nwqid, wqid[16] }; partial walk → error.Nine with ename "not found"-ish - pub fn clone(s, fid: u32) Error!u32; // allocFid + walk with no names - pub fn open(s, fid: u32, mode: u8) Error!Open; // Open = { qid, iounit } - pub fn create(s, fid: u32, name: []const u8, perm: u32, mode: u8) Error!Open; - pub fn read(s, fid: u32, offset: u64, buf: []u8) Error!usize; // chunks by maxRead/iounit; stops at short read - pub fn write(s, fid: u32, offset: u64, data: []const u8) Error!usize; // chunks; stops at short write - pub fn stat(s, fid: u32) Error!cloud9.Stat; // strings borrow the input buffer - pub fn wstat(s, fid: u32, st: cloud9.Stat) Error!void; - pub fn clunk(s, fid: u32) Error!void; // frees the fid even on error - pub fn remove(s, fid: u32) Error!void; // frees the fid even on error - pub fn errno(s: *const Session) std.os.linux.E; // map ename → errno (see below) + pub fn version(c) Error!void; + pub fn attach(c, fid: u32, uname: []const u8, aname: []const u8) Error!cloud9.Qid; + pub fn walk(c, fid: u32, newfid: u32, names: []const []const u8) Error!Walk; // Walk = { nwqid, wqid[16] }; partial walk → error.Nine with ename "not found"-ish + pub fn clone(c, fid: u32) Error!u32; // allocFid + walk with no names + pub fn open(c, fid: u32, mode: u8) Error!Open; // Open = { qid, iounit } + pub fn create(c, fid: u32, name: []const u8, perm: u32, mode: u8) Error!Open; + pub fn read(c, fid: u32, offset: u64, buf: []u8) Error!usize; // chunks by maxRead/iounit; stops at short read + pub fn write(c, fid: u32, offset: u64, data: []const u8) Error!usize; // chunks; stops at short write + pub fn stat(c, fid: u32) Error!cloud9.Stat; // strings borrow the reply buffer + pub fn wstat(c, fid: u32, st: cloud9.Stat) Error!void; + pub fn clunk(c, fid: u32) Error!void; // frees the fid even on error + pub fn remove(c, fid: u32) Error!void; // frees the fid even on error + pub fn errno(c: *const Caller) std.os.linux.E; // map ename → errno (see below) }; pub const dontcare = cloud9.Stat{ .type = 0xFFFF, .dev = 0xFFFF_FFFF, .qid = .{ .type = 0xFF, .version = 0xFFFF_FFFF, .path = 0xFFFF_FFFF_FFFF_FFFF }, .mode = 0xFFFF_FFFF, .atime = 0xFFFF_FFFF, .mtime = 0xFFFF_FFFF, .length = 0xFFFF_FFFF_FFFF_FFFF, .name = "", .uid = "", .gid = "", .muid = "" }; ``` `connect`: for `.unix` and `.tcp` create a blocking `SOCK_STREAM|SOCK_CLOEXEC` socket and connect (`TCP_NODELAY` on TCP); for `.fd` adopt it. Buffers of -`msize` bytes for in/out are heap allocated. Then submit `.version`, drain -output to the socket, read until `take()` yields the version result. The -negotiated msize is `result.version.msize`; if the server answered -`"unknown"`, fail with `error.Protocol`. +`msize` bytes for in/out are heap allocated. Then a `version` rpc on a +temporary caller. The negotiated msize is `result.version.msize`; if the +server answered `"unknown"`, fail with `error.Protocol`. -`rpc`: submit, write all of `client.output()` (calling `wrote`), then loop: -`take()`; if null, poll the fd together with `stop_fd` and the interrupt -source's descriptor; when the fd is readable, `read` into a temp buffer and -`push` (push returns how much fit; the frame is at most msize so it always -fits after a `take`). If the fd returns 0 → `error.Closed`. If the client -dies → `error.Protocol`. `stop_fd` readable → `error.Stopped`. When the -interrupt source asks for it, submit `.flush = .{ .oldtag = tag }` and keep -waiting: the original reply → returned normally (the Rflush that follows is -skipped by a later call); the Rflush first → `error.Interrupted`. A `.fail` -result copies the ename and returns `error.Nine`. `read`/`write` chunk loops -stop between chunks once the source reports `armed`, and turn an +`rpc` multiplexes the way Plan 9's mount driver does +(`kernel/devices/sys/devmnt.c` in principia-softwarica: `mountio`, +`mountmux`, `mntgate`, `mntflushalloc`). Under the session's `wlock` +(which serializes the client's output buffer and the socket's write side) +and `mutex` (tags, input side, the waiter table) the call claims a tag, +registers its waiter under it and writes the T-message; the write itself +runs with only `wlock` held. Then it waits: the first waiter that finds +nobody reading takes the **reading baton** (`m->rip`), reads the socket, +`push`es and `take`s, copies each R-message into the caller its tag names +(the client's input buffer belongs to whoever reads next) and pokes that +caller's eventfd; when its own reply is in it lets go of the baton and +pokes every waiter, one of which takes it over. Waiters without the baton +poll their eventfd; all poll `stop_fd` (→ `error.Stopped`) and their own +interrupt source. A socket EOF → `error.Closed`, a client protocol death → +`error.Protocol`, for every waiter. When all `nine.max_tags` tags are +taken a call waits on a starved list (reading if it gets the baton) until a +reply frees one; nothing fails for lack of a tag. + +When the interrupt source asks for it the waiter first gives up the baton +(a Tflush write that blocks must not stop everyone's replies, the Rflush +among them, being read), then submits `.flush = .{ .oldtag = tag }` under a +tag of its own and keeps waiting on both tags: the original reply first → +it is honoured and returned (flush(5): a Twrite that completed changed +state; the Rflush that follows is dropped); the Rflush first → +`error.Interrupted`. The client does not reuse the flushed tag until the +Rflush is in. A Tflush unanswered for `flush_grace_ms` wedges the session +(every waiter → `error.Closed`). A `.fail` result copies the ename into the +caller and returns `error.Nine`. `read`/`write` chunk loops (per call, per +caller) stop between chunks once the source reports `armed`, and turn an `Interrupted` chunk into a short count when earlier chunks moved data. Rerror text → errno mapping (case-insensitive substring, in this order): @@ -491,8 +522,8 @@ Op mapping (9P2000 has no symlinks, links, xattrs, locks, mknod): | SETATTR | `stat` then `wstat` with a *dontcare* Stat: `FATTR_SIZE`→length; `FATTR_MODE`→`(old.mode & ~0o777) \| (mode & 0o777)`; `FATTR_MTIME`→mtime (`FATTR_MTIME_NOW` → now); `FATTR_ATIME` ignored; `FATTR_UID/GID` → `EPERM` unless unchanged; then `stat` again for the reply | | OPEN | `clone(inode.fid)` then `open(newfid, mode)`; mode from `O_ACCMODE` (`oread/owrite/ordwr`), `O_TRUNC` → `otrunc`; reply `OpenOut{ fh, open_flags = FOPEN_DIRECT_IO }`; on failure clunk | | OPENDIR | same with `oread`; `fh` with `dir = null` | -| READ | `read(fh.fid, offset, buf[0..min(size, 1 MiB)])`; reply data | -| WRITE | `write(fh.fid, offset, data)`; `WriteOut{ size = n }` | +| READ | on a transfer worker: `read(fh.fid, offset, buf[0..min(size, 1 MiB)])`; reply data | +| WRITE | on a transfer worker: `write(fh.fid, offset, data)`; `WriteOut{ size = n }` | | READDIR | fill `Dirent`s from the `DirList` starting at `offset`, up to `size` bytes | | RELEASE / RELEASEDIR | `clunk(fh.fid)`; free DirList | | FLUSH / FSYNC / FSYNCDIR | ok (no-op) | @@ -528,12 +559,46 @@ rather than poisoning the whole READDIR reply; `length` near 2^64 is clamped to `i64` max; the errno of a failing 9P call is latched before any cleanup clunk overwrites the session's ename. +#### Concurrency (transfer workers) + +One thread, the loop, owns the inode, handle and qid tables and serves +every request that reads or changes them, one at a time; lookups and +forgets are reconciled there and nowhere else. READ and WRITE are the +requests a synthetic server parks (a blocking `log`, `event` or `run` +file), and they need no table beyond the handle → fid lookup, so the loop +does that lookup and hands the transfer to a **transfer worker** +(`Pool`/`Worker` in `bridge.zig`): a thread with its own `nine.Caller`, +data buffer and interrupt pipe, which makes the 9P calls, chunks by +msize/iounit itself and writes the reply to the FUSE fd. The session +multiplexes their tags, so a parked read holds up only its own process: +with pardes's `log` parked, `cat status` answers in milliseconds instead of +waiting for the read. Workers start on demand, up to `max_workers = +nine.max_tags - 1` per bridge (the loop keeps the last tag); a transfer +that finds every worker parked waits in the queue for one, where an +INTERRUPT answers it `EINTR` at once. Every queued transfer gets an idle +worker of its own or a new one, so a transfer never waits behind a parked +one while the pool has room. + +The fid a worker uses is the handle's, and is clunked only on RELEASE, +which the kernel sends once no read or write on that open file is in +progress (FLUSH, sent on every `close`, clunks nothing). Two processes +reading or writing one open file concurrently now run concurrently on +the server too; one process's write-then-read stays ordered because each +syscall waits for its reply, and 9P promises nothing more. + +This is devmnt's model with the waiting given threads: in Plan 9 every +process blocked in a read is its own waiter on the mount; a FUSE server +gets the kernel's requests on one descriptor, so the waiting processes +are represented by workers here. + #### Interrupts (FUSE_INTERRUPT → Tflush) -One request at a time, but not deaf. `serve` puts the FUSE fd in `O_NONBLOCK` -mode (every read follows a poll) and installs a `nine.Interrupt` source that -`Session.rpc` polls together with the socket and `stop_fd` whenever a reply -is outstanding, including the initial root stat: +The loop waits on the server too (its own lookups, opens, stats), but it +is not deaf meanwhile. `serve` puts the FUSE fd in `O_NONBLOCK` mode (every +read follows a poll) and installs a `nine.Interrupt` source on the loop's +caller that `Caller.rpc` polls together with `stop_fd` (and the socket, +when it holds the baton) whenever a reply is outstanding, including the +initial root stat: * `onReadable` reads the request the kernel has ready into a second 8-aligned buffer of `request_buf_len` bytes (`spare_buf`). A @@ -541,15 +606,21 @@ is outstanding, including the initial root stat: (`cur_unique`) sets `interrupted` and returns true: `rpc` sends `Tflush(oldtag)` and keeps waiting until either the original reply arrives (the interrupt raced it: the result is returned as if nothing happened and - the Rflush that follows is swallowed by a later call) or the Rflush does + the Rflush that follows is dropped by whoever reads it) or the Rflush does (`error.Interrupted` → `EINTR`; a server that instead answers the flushed request with an Rerror containing "interrupt", as Pardes does, lands on the - same errno through the ename table). An INTERRUPT for any other unique is - consumed and dropped (the kernel expects no reply). Any other request + same errno through the ename table). An INTERRUPT for a transfer worker's + request goes to that worker: still queued, it is answered `EINTR` on the + spot; in flight, its unique goes down the worker's interrupt pipe, and the + worker's session sends the Tflush the same way. An INTERRUPT for any + other unique is consumed and dropped (the kernel expects no reply); the + loop routes one it reads while idle the same way. Any other request (FORGET, RELEASE, INIT during the root stat, a second process's LOOKUP) is copied into a queue that `serve` dispatches, in arrival order, before it - polls again; the fd stays watched throughout, so an INTERRUPT is never - stuck behind a parked request (a copy that cannot be allocated answers + polls again (a READ or WRITE goes to a worker on the spot instead, and an + INTERRUPT for a request still in that queue answers it `EINTR`); the fd + stays watched throughout, so an INTERRUPT is never stuck behind a parked + request (a copy that cannot be allocated answers `ENOMEM` on the spot). * The INTERRUPT applies to `cur_unique` only and is consumed when read: the clunks that unwind a half-done lookup/create/mkdir after an `Interrupted` @@ -559,7 +630,7 @@ is outstanding, including the initial root stat: * Multi-step operations fail with `EINTR` at whichever rpc was flushed and release the fids they had allocated (`--debug` prints `fids=N` per request; the interrupt suite checks it and the server-side count). The - chunk loops (`Session.read/write`, `loadDir`) also stop between chunks + chunk loops (`Caller.read/write`, `loadDir`) also stop between chunks while `armed`, so a reply that won the race cannot lead into another blocking chunk: a read or write that already moved data returns the partial count like read(2), one that moved nothing and a directory listing return @@ -575,7 +646,32 @@ behind — so the reader gets `EINTR` and the session ends (the hostile `never` mode; in single-connection mode that is the mount, in mntgen that one name). A slow-but-alive server that cannot answer a flush inside its own blocking read pays the same price; the alternative was a hang that only -SIGKILL of 9ns could end. +SIGKILL of 9ns could end. When a worker's flush wedges the session the +loop notices at its next request (every call on the session then fails) +and the mount ends there. + +Budgets: a session's 9P client is `cloud9.ClientWith(nine.max_tags)`, 32 +request tags plus one kept for flushes (cloud9's default `Client` has 16), +so one mount has at most 32 requests on the server at once. 32 is the +cloud9 file server engine's `slot_capacity` (`src/fs.zig`, what pardes +serves with): the most reads, writes, opens and clunks it parks per +connection, answering the one past it with Rerror "Resource temporarily +unavailable" (which would reach the reader as `EIO`). A mount can +therefore never meet that refusal. The two are raised together or not at +all: the engine's default is also the ESP32 board's (its budget tests in +`fs.zig` hold a default server to 24 KiB), so a bigger window is a +per-server option (pardes's editor `fs.Options`) plus `nine.max_tags`, +not a new default. + +When the window is full: the loop keeps a tag of its own, so lookups, +opens, stats, listings and every other request the loop serves go on; +31 parked transfers occupy all the workers, and a 32nd READ or WRITE, +quick or not (9ns cannot tell a `status` read from a `log` read), waits +in the pool queue until a parked one ends or is interrupted. It blocks; +it is never refused (`EAGAIN`) or failed (`EIO`), and an INTERRUPT +answers it `EINTR` at once. A worker whose flushed request was answered +before its Rflush holds the flushed tag until that Rflush; a call that +meanwhile finds no tag waits for one the same way. ### `src/ns.zig` — namespace and process plumbing |
