summaryrefslogtreecommitdiff
path: root/9ns/docs
diff options
context:
space:
mode:
Diffstat (limited to '9ns/docs')
-rw-r--r--9ns/docs/DESIGN.md208
1 files changed, 152 insertions, 56 deletions
diff --git a/9ns/docs/DESIGN.md b/9ns/docs/DESIGN.md
index 32c548c..ad59521 100644
--- a/9ns/docs/DESIGN.md
+++ b/9ns/docs/DESIGN.md
@@ -166,12 +166,13 @@ Threads and node ids:
* A **mount** is a posted name walked into: the socket to dial, its own
`nine.Session` and bridge state (inode table, handles — the ordinary
single-server translation, unchanged) once dialed, and a **worker
- thread** that owns all of it. The dispatcher makes a mount without
+ thread** that owns all of it (its transfer workers share the session). The dispatcher makes a mount without
touching the network and hands it the walk; the worker dials while
serving that first request. It pops copied requests off a queue and
- serves them one at a time, exactly the single-connection contract; the
- dispatcher keeps reading `/dev/fuse` meanwhile, so one slow server never
- blocks the other names. Replies go straight back on the FUSE fd (one
+ serves them one at a time, exactly the single-connection contract
+ (READ and WRITE handed on to that mount's transfer workers, see
+ *Concurrency* under `src/bridge.zig`); the dispatcher keeps reading
+ `/dev/fuse` meanwhile, so one slow server never blocks the other names. Replies go straight back on the FUSE fd (one
`writev` per reply; the kernel processes each write as one message).
The queue is a plain list under a mutex and condition rather than an
`std.Io.Queue` because an interrupt has to find and remove a request by
@@ -353,15 +354,17 @@ pub fn nameAfter(comptime T: type, req: Request) ![]const u8; // NUL-terminated
The request buffer must be ≥ `max_write + 4096`; 9ns uses 1 MiB + 4 KiB.
Requests with an unknown/unsupported opcode get `ENOSYS`.
-### `src/nine.zig` — synchronous 9P2000 session on a blocking fd
+### `src/nine.zig` — 9P2000 session on a blocking fd, shared by threads
-Thin, synchronous RPC layer over `cloud9.Client` (which is push/take,
-non-blocking-agnostic). One outstanding request at a time (the FUSE loop is
-single-threaded). Fids are allocated from a free list.
+Thin RPC layer over `cloud9.Client` (which is push/take,
+non-blocking-agnostic). Many requests may be in flight at once, each under
+its own tag; replies come back in any order. The connection is a `Session`;
+each thread that makes calls on it has a `Caller`. Fids are allocated from
+a free list (under the session's mutex).
```zig
pub const Address = union(enum) { unix: []const u8, tcp: struct { host: []const u8, port: u16 }, fd: i32 };
-/// Owner's hook into the reply wait (the bridge's FUSE fd); see "Interrupts" under bridge.zig.
+/// Owner's hook into the reply wait (the bridge's FUSE fd, or an interrupt pipe); see "Interrupts" under bridge.zig.
pub const Interrupt = struct {
ctx: *anyopaque,
watch: *const fn (ctx) i32, // fd to poll alongside the socket, or -1
@@ -370,53 +373,81 @@ pub const Interrupt = struct {
};
pub const Session = struct {
pub const Error = error{ Nine, Protocol, Io, Closed, Stopped, Interrupted, TooLarge, OutOfMemory };
- /// After error.Nine, `ename` holds the server's Rerror text (copied, bounded).
- ename: [256]u8, ename_len: usize,
msize: u32,
- stop_fd: i32 = -1, // readable → pending rpc fails with error.Stopped
- interrupt: ?Interrupt = null,
+ stop_fd: i32 = -1, // readable → every waiting call fails with error.Stopped
+ flush_grace_ms: i32 = 3000, // a Tflush unanswered this long wedges the session
+ wedged: bool, // declared gone: every call → error.Closed
pub fn connect(gpa: std.mem.Allocator, address: Address, msize: u32) !Session; // socket+connect, version
- pub fn deinit(s: *Session) void;
- pub fn attach(s: *Session, fid: u32, uname: []const u8, aname: []const u8) Error!cloud9.Qid;
+ pub fn deinit(s: *Session) void; // after every Caller is done
pub fn allocFid(s: *Session) u32;
pub fn freeFid(s: *Session, fid: u32) void;
- /// Generic RPC. Result slices borrow the input buffer until the next call.
- pub fn rpc(s: *Session, req: cloud9.Client.Request) Error!cloud9.Client.Result;
+ pub fn over(s: *Session) bool; // wedged, or the connection died
+};
+pub const Caller = struct {
+ session: *Session,
+ /// After error.Nine, `ename` holds the server's Rerror text (copied, bounded).
+ ename: [256]u8, ename_len: usize,
+ interrupt: ?Interrupt = null,
+ abort_on_cancel: bool = false, // handshake: a cancellation hangs up instead of flushing
+
+ pub fn init(s: *Session) !Caller; // msize reply buffer + wake-up eventfd
+ pub fn deinit(c: *Caller) void;
+ /// Generic RPC. Result slices borrow the caller's reply buffer until its next call.
+ pub fn rpc(c: *Caller, req: cloud9.Client.Request) Error!cloud9.Client.Result;
// Conveniences (all built on rpc):
- pub fn walk(s, fid: u32, newfid: u32, names: []const []const u8) Error!Walk; // Walk = { nwqid, wqid[16] }; partial walk → error.Nine with ename "not found"-ish
- pub fn clone(s, fid: u32) Error!u32; // allocFid + walk with no names
- pub fn open(s, fid: u32, mode: u8) Error!Open; // Open = { qid, iounit }
- pub fn create(s, fid: u32, name: []const u8, perm: u32, mode: u8) Error!Open;
- pub fn read(s, fid: u32, offset: u64, buf: []u8) Error!usize; // chunks by maxRead/iounit; stops at short read
- pub fn write(s, fid: u32, offset: u64, data: []const u8) Error!usize; // chunks; stops at short write
- pub fn stat(s, fid: u32) Error!cloud9.Stat; // strings borrow the input buffer
- pub fn wstat(s, fid: u32, st: cloud9.Stat) Error!void;
- pub fn clunk(s, fid: u32) Error!void; // frees the fid even on error
- pub fn remove(s, fid: u32) Error!void; // frees the fid even on error
- pub fn errno(s: *const Session) std.os.linux.E; // map ename → errno (see below)
+ pub fn version(c) Error!void;
+ pub fn attach(c, fid: u32, uname: []const u8, aname: []const u8) Error!cloud9.Qid;
+ pub fn walk(c, fid: u32, newfid: u32, names: []const []const u8) Error!Walk; // Walk = { nwqid, wqid[16] }; partial walk → error.Nine with ename "not found"-ish
+ pub fn clone(c, fid: u32) Error!u32; // allocFid + walk with no names
+ pub fn open(c, fid: u32, mode: u8) Error!Open; // Open = { qid, iounit }
+ pub fn create(c, fid: u32, name: []const u8, perm: u32, mode: u8) Error!Open;
+ pub fn read(c, fid: u32, offset: u64, buf: []u8) Error!usize; // chunks by maxRead/iounit; stops at short read
+ pub fn write(c, fid: u32, offset: u64, data: []const u8) Error!usize; // chunks; stops at short write
+ pub fn stat(c, fid: u32) Error!cloud9.Stat; // strings borrow the reply buffer
+ pub fn wstat(c, fid: u32, st: cloud9.Stat) Error!void;
+ pub fn clunk(c, fid: u32) Error!void; // frees the fid even on error
+ pub fn remove(c, fid: u32) Error!void; // frees the fid even on error
+ pub fn errno(c: *const Caller) std.os.linux.E; // map ename → errno (see below)
};
pub const dontcare = cloud9.Stat{ .type = 0xFFFF, .dev = 0xFFFF_FFFF, .qid = .{ .type = 0xFF, .version = 0xFFFF_FFFF, .path = 0xFFFF_FFFF_FFFF_FFFF }, .mode = 0xFFFF_FFFF, .atime = 0xFFFF_FFFF, .mtime = 0xFFFF_FFFF, .length = 0xFFFF_FFFF_FFFF_FFFF, .name = "", .uid = "", .gid = "", .muid = "" };
```
`connect`: for `.unix` and `.tcp` create a blocking `SOCK_STREAM|SOCK_CLOEXEC`
socket and connect (`TCP_NODELAY` on TCP); for `.fd` adopt it. Buffers of
-`msize` bytes for in/out are heap allocated. Then submit `.version`, drain
-output to the socket, read until `take()` yields the version result. The
-negotiated msize is `result.version.msize`; if the server answered
-`"unknown"`, fail with `error.Protocol`.
+`msize` bytes for in/out are heap allocated. Then a `version` rpc on a
+temporary caller. The negotiated msize is `result.version.msize`; if the
+server answered `"unknown"`, fail with `error.Protocol`.
-`rpc`: submit, write all of `client.output()` (calling `wrote`), then loop:
-`take()`; if null, poll the fd together with `stop_fd` and the interrupt
-source's descriptor; when the fd is readable, `read` into a temp buffer and
-`push` (push returns how much fit; the frame is at most msize so it always
-fits after a `take`). If the fd returns 0 → `error.Closed`. If the client
-dies → `error.Protocol`. `stop_fd` readable → `error.Stopped`. When the
-interrupt source asks for it, submit `.flush = .{ .oldtag = tag }` and keep
-waiting: the original reply → returned normally (the Rflush that follows is
-skipped by a later call); the Rflush first → `error.Interrupted`. A `.fail`
-result copies the ename and returns `error.Nine`. `read`/`write` chunk loops
-stop between chunks once the source reports `armed`, and turn an
+`rpc` multiplexes the way Plan 9's mount driver does
+(`kernel/devices/sys/devmnt.c` in principia-softwarica: `mountio`,
+`mountmux`, `mntgate`, `mntflushalloc`). Under the session's `wlock`
+(which serializes the client's output buffer and the socket's write side)
+and `mutex` (tags, input side, the waiter table) the call claims a tag,
+registers its waiter under it and writes the T-message; the write itself
+runs with only `wlock` held. Then it waits: the first waiter that finds
+nobody reading takes the **reading baton** (`m->rip`), reads the socket,
+`push`es and `take`s, copies each R-message into the caller its tag names
+(the client's input buffer belongs to whoever reads next) and pokes that
+caller's eventfd; when its own reply is in it lets go of the baton and
+pokes every waiter, one of which takes it over. Waiters without the baton
+poll their eventfd; all poll `stop_fd` (→ `error.Stopped`) and their own
+interrupt source. A socket EOF → `error.Closed`, a client protocol death →
+`error.Protocol`, for every waiter. When all `nine.max_tags` tags are
+taken a call waits on a starved list (reading if it gets the baton) until a
+reply frees one; nothing fails for lack of a tag.
+
+When the interrupt source asks for it the waiter first gives up the baton
+(a Tflush write that blocks must not stop everyone's replies, the Rflush
+among them, being read), then submits `.flush = .{ .oldtag = tag }` under a
+tag of its own and keeps waiting on both tags: the original reply first →
+it is honoured and returned (flush(5): a Twrite that completed changed
+state; the Rflush that follows is dropped); the Rflush first →
+`error.Interrupted`. The client does not reuse the flushed tag until the
+Rflush is in. A Tflush unanswered for `flush_grace_ms` wedges the session
+(every waiter → `error.Closed`). A `.fail` result copies the ename into the
+caller and returns `error.Nine`. `read`/`write` chunk loops (per call, per
+caller) stop between chunks once the source reports `armed`, and turn an
`Interrupted` chunk into a short count when earlier chunks moved data.
Rerror text → errno mapping (case-insensitive substring, in this order):
@@ -491,8 +522,8 @@ Op mapping (9P2000 has no symlinks, links, xattrs, locks, mknod):
| SETATTR | `stat` then `wstat` with a *dontcare* Stat: `FATTR_SIZE`→length; `FATTR_MODE`→`(old.mode & ~0o777) \| (mode & 0o777)`; `FATTR_MTIME`→mtime (`FATTR_MTIME_NOW` → now); `FATTR_ATIME` ignored; `FATTR_UID/GID` → `EPERM` unless unchanged; then `stat` again for the reply |
| OPEN | `clone(inode.fid)` then `open(newfid, mode)`; mode from `O_ACCMODE` (`oread/owrite/ordwr`), `O_TRUNC` → `otrunc`; reply `OpenOut{ fh, open_flags = FOPEN_DIRECT_IO }`; on failure clunk |
| OPENDIR | same with `oread`; `fh` with `dir = null` |
-| READ | `read(fh.fid, offset, buf[0..min(size, 1 MiB)])`; reply data |
-| WRITE | `write(fh.fid, offset, data)`; `WriteOut{ size = n }` |
+| READ | on a transfer worker: `read(fh.fid, offset, buf[0..min(size, 1 MiB)])`; reply data |
+| WRITE | on a transfer worker: `write(fh.fid, offset, data)`; `WriteOut{ size = n }` |
| READDIR | fill `Dirent`s from the `DirList` starting at `offset`, up to `size` bytes |
| RELEASE / RELEASEDIR | `clunk(fh.fid)`; free DirList |
| FLUSH / FSYNC / FSYNCDIR | ok (no-op) |
@@ -528,12 +559,46 @@ rather than poisoning the whole READDIR reply; `length` near 2^64 is clamped
to `i64` max; the errno of a failing 9P call is latched before any cleanup
clunk overwrites the session's ename.
+#### Concurrency (transfer workers)
+
+One thread, the loop, owns the inode, handle and qid tables and serves
+every request that reads or changes them, one at a time; lookups and
+forgets are reconciled there and nowhere else. READ and WRITE are the
+requests a synthetic server parks (a blocking `log`, `event` or `run`
+file), and they need no table beyond the handle → fid lookup, so the loop
+does that lookup and hands the transfer to a **transfer worker**
+(`Pool`/`Worker` in `bridge.zig`): a thread with its own `nine.Caller`,
+data buffer and interrupt pipe, which makes the 9P calls, chunks by
+msize/iounit itself and writes the reply to the FUSE fd. The session
+multiplexes their tags, so a parked read holds up only its own process:
+with pardes's `log` parked, `cat status` answers in milliseconds instead of
+waiting for the read. Workers start on demand, up to `max_workers =
+nine.max_tags - 1` per bridge (the loop keeps the last tag); a transfer
+that finds every worker parked waits in the queue for one, where an
+INTERRUPT answers it `EINTR` at once. Every queued transfer gets an idle
+worker of its own or a new one, so a transfer never waits behind a parked
+one while the pool has room.
+
+The fid a worker uses is the handle's, and is clunked only on RELEASE,
+which the kernel sends once no read or write on that open file is in
+progress (FLUSH, sent on every `close`, clunks nothing). Two processes
+reading or writing one open file concurrently now run concurrently on
+the server too; one process's write-then-read stays ordered because each
+syscall waits for its reply, and 9P promises nothing more.
+
+This is devmnt's model with the waiting given threads: in Plan 9 every
+process blocked in a read is its own waiter on the mount; a FUSE server
+gets the kernel's requests on one descriptor, so the waiting processes
+are represented by workers here.
+
#### Interrupts (FUSE_INTERRUPT → Tflush)
-One request at a time, but not deaf. `serve` puts the FUSE fd in `O_NONBLOCK`
-mode (every read follows a poll) and installs a `nine.Interrupt` source that
-`Session.rpc` polls together with the socket and `stop_fd` whenever a reply
-is outstanding, including the initial root stat:
+The loop waits on the server too (its own lookups, opens, stats), but it
+is not deaf meanwhile. `serve` puts the FUSE fd in `O_NONBLOCK` mode (every
+read follows a poll) and installs a `nine.Interrupt` source on the loop's
+caller that `Caller.rpc` polls together with `stop_fd` (and the socket,
+when it holds the baton) whenever a reply is outstanding, including the
+initial root stat:
* `onReadable` reads the request the kernel has ready into a second
8-aligned buffer of `request_buf_len` bytes (`spare_buf`). A
@@ -541,15 +606,21 @@ is outstanding, including the initial root stat:
(`cur_unique`) sets `interrupted` and returns true: `rpc` sends
`Tflush(oldtag)` and keeps waiting until either the original reply arrives
(the interrupt raced it: the result is returned as if nothing happened and
- the Rflush that follows is swallowed by a later call) or the Rflush does
+ the Rflush that follows is dropped by whoever reads it) or the Rflush does
(`error.Interrupted` → `EINTR`; a server that instead answers the flushed
request with an Rerror containing "interrupt", as Pardes does, lands on the
- same errno through the ename table). An INTERRUPT for any other unique is
- consumed and dropped (the kernel expects no reply). Any other request
+ same errno through the ename table). An INTERRUPT for a transfer worker's
+ request goes to that worker: still queued, it is answered `EINTR` on the
+ spot; in flight, its unique goes down the worker's interrupt pipe, and the
+ worker's session sends the Tflush the same way. An INTERRUPT for any
+ other unique is consumed and dropped (the kernel expects no reply); the
+ loop routes one it reads while idle the same way. Any other request
(FORGET, RELEASE, INIT during the root stat, a second process's LOOKUP) is
copied into a queue that `serve` dispatches, in arrival order, before it
- polls again; the fd stays watched throughout, so an INTERRUPT is never
- stuck behind a parked request (a copy that cannot be allocated answers
+ polls again (a READ or WRITE goes to a worker on the spot instead, and an
+ INTERRUPT for a request still in that queue answers it `EINTR`); the fd
+ stays watched throughout, so an INTERRUPT is never stuck behind a parked
+ request (a copy that cannot be allocated answers
`ENOMEM` on the spot).
* The INTERRUPT applies to `cur_unique` only and is consumed when read: the
clunks that unwind a half-done lookup/create/mkdir after an `Interrupted`
@@ -559,7 +630,7 @@ is outstanding, including the initial root stat:
* Multi-step operations fail with `EINTR` at whichever rpc was flushed and
release the fids they had allocated (`--debug` prints `fids=N` per
request; the interrupt suite checks it and the server-side count). The
- chunk loops (`Session.read/write`, `loadDir`) also stop between chunks
+ chunk loops (`Caller.read/write`, `loadDir`) also stop between chunks
while `armed`, so a reply that won the race cannot lead into another
blocking chunk: a read or write that already moved data returns the partial
count like read(2), one that moved nothing and a directory listing return
@@ -575,7 +646,32 @@ behind — so the reader gets `EINTR` and the session ends (the hostile
`never` mode; in single-connection mode that is the mount, in mntgen that
one name). A slow-but-alive server that cannot answer a flush inside its own
blocking read pays the same price; the alternative was a hang that only
-SIGKILL of 9ns could end.
+SIGKILL of 9ns could end. When a worker's flush wedges the session the
+loop notices at its next request (every call on the session then fails)
+and the mount ends there.
+
+Budgets: a session's 9P client is `cloud9.ClientWith(nine.max_tags)`, 32
+request tags plus one kept for flushes (cloud9's default `Client` has 16),
+so one mount has at most 32 requests on the server at once. 32 is the
+cloud9 file server engine's `slot_capacity` (`src/fs.zig`, what pardes
+serves with): the most reads, writes, opens and clunks it parks per
+connection, answering the one past it with Rerror "Resource temporarily
+unavailable" (which would reach the reader as `EIO`). A mount can
+therefore never meet that refusal. The two are raised together or not at
+all: the engine's default is also the ESP32 board's (its budget tests in
+`fs.zig` hold a default server to 24 KiB), so a bigger window is a
+per-server option (pardes's editor `fs.Options`) plus `nine.max_tags`,
+not a new default.
+
+When the window is full: the loop keeps a tag of its own, so lookups,
+opens, stats, listings and every other request the loop serves go on;
+31 parked transfers occupy all the workers, and a 32nd READ or WRITE,
+quick or not (9ns cannot tell a `status` read from a `log` read), waits
+in the pool queue until a parked one ends or is interrupted. It blocks;
+it is never refused (`EAGAIN`) or failed (`EIO`), and an INTERRUPT
+answers it `EINTR` at once. A worker whose flushed request was answered
+before its Rflush holds the flushed tag until that Rflush; a call that
+meanwhile finds no tag waits for one the same way.
### `src/ns.zig` — namespace and process plumbing