From 9dd6f001c7d5eee987cbd305a79d1b1d011b83f5 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Mon, 12 Oct 2026 17:23:34 -0300 Subject: 9web: backpressure asynchronous replies under parallel site load Reserve output space for an async completion plus the already-dispatched synchronous handler. Wait on a condition without the catalogue mutex, preserving cancellation and connection-slot lifetime. Abort HTTP streaming on an upstream error instead of hiding a truncated response. Reproduced the former failure with concurrent live cafe assets. Add exact-hash load checks with 16 readers, mixed HEAD and 40 interrupted downloads; validate 80 multi-frame responses over 37 live files. All five live sites, mounted 9P browsing, protocol/catalogue/host/HTTP/mux/UI tests and 32 original E2E scenarios pass. Deploy corrected service on sole [::]:8090. See docs/namespace-sites-validation.md. --- docs/catalog.md | 3 +++ docs/namespace-sites-validation.md | 28 ++++++++++++++++++++++++ src/serve.zig | 29 +++++++++++++++++++++++++ test/web/catalog_ui.mjs | 4 ++++ test/web/parallel_reads.mjs | 44 ++++++++++++++++++++++++++++++++++++++ web/catalog.zig | 15 ++++++------- web/httpfs.zig | 4 +++- 7 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 test/web/parallel_reads.mjs diff --git a/docs/catalog.md b/docs/catalog.md index bc308c2..d8757b0 100644 --- a/docs/catalog.md +++ b/docs/catalog.md @@ -287,6 +287,9 @@ control writes, non-seekable EOF, blocking FUSE reads concurrent with other requests, and 40 flush/clunk cycles without exhausting background readers. The Chromium test navigates a real mounted fixture, reads a 180 KB file and opens/stops a live event stream directly from its directory entry. +It also checks exact multi-frame bytes under parallel load, alongside canceled +downloads and synchronous metadata requests. Asynchronous host-read replies +wait for output space rather than overfilling the shared 9P connection. ## Publication and namespace policy diff --git a/docs/namespace-sites-validation.md b/docs/namespace-sites-validation.md index 64858fc..e73a572 100644 --- a/docs/namespace-sites-validation.md +++ b/docs/namespace-sites-validation.md @@ -72,3 +72,31 @@ IPv6 site requests return 200. An actual café build requested through prefixed generation; no preview daemon was started. `zig build -Doptimize=ReleaseSafe e2e` passes all 32 original gateway scenarios. + +## Parallel-read follow-up + +A final cold browser run exposed an intermittent load failure despite earlier +successful runs. Sixteen concurrent large-asset requests reproduced HTTP 502s: +independent host-read completions could overfill the runner's bounded 9P output +queue. The synchronous dispatch path already checked output space, but these +asynchronous completions bypassed it. + +`Runner.Conn.replyAsync` now waits on an output-space condition, reserving one +additional full frame for an already-dispatched synchronous handler. It waits +without the catalogue mutex, remains cancellable, and is joined before slot +reuse. HTTP streaming now aborts an upstream read error instead of ending a +truncated response successfully. + +`test/web/parallel_reads.mjs` checks 80 exact multi-frame responses with 16 +simultaneous readers, 40 interrupted downloads and mixed HEAD requests. It is +part of the mounted-host browser fixture and can also be run read-only against +the live café: + +``` +node test/web/parallel_reads.mjs http://127.0.0.1:8090/sites/cafe/ \ + /home/goblin/.local/state/9web/public/cafe +``` + +The deployed service passes this load test against 37 café source files, followed +by the complete live browser traversal again. Protocol/unit, catalogue, mounted +host, HTTP FS, mux, UI and original 32-scenario E2E checks pass after the fix. diff --git a/src/serve.zig b/src/serve.zig index ad774a3..7ee47ba 100644 --- a/src/serve.zig +++ b/src/serve.zig @@ -123,6 +123,7 @@ pub fn Runner(comptime Backend: type, comptime opts: fs.Options, comptime limits flags: std.atomic.Value(u8) = .init(0), signal: Io.Event = .unset, room: Io.Event = .unset, + output_room: Io.Condition = .init, used: std.atomic.Value(bool) = .init(false), in: [msize]u8 = undefined, out: [2 * msize]u8 = undefined, @@ -152,6 +153,32 @@ pub fn Runner(comptime Backend: type, comptime opts: fs.Options, comptime limits c.raise(output); } + /// Completion from an independent task. Keep one full frame of + /// output capacity reserved for the synchronous handler that the + /// connection loop may already have dispatched. Waiting must not + /// hold a backend lock: that handler must be able to finish so the + /// connection loop can drain output. Cancellation/clunk must join + /// the producer task before this connection slot can be reused. + pub fn replyAsync(c: *Conn, r: *const Backend.Reply, bytes: []const u8) Io.Cancelable!void { + const io = c.runner.io; + c.lock(); + defer c.unlock(); + while (true) { + if (c.engine.protocol.dead or c.runner.stopping.load(.acquire)) { + return; + } + _ = c.engine.protocol.hasRoom(); // compact drained prefix + const reserve = 2 * @as(usize, @max(c.engine.protocol.msize, 24)); + if (c.engine.protocol.out.len - c.engine.protocol.out_len >= reserve) { + c.engine.reply(r, bytes); + c.raise(output); + return; + } + c.raise(output); + try c.output_room.wait(io, &c.mutex); + } + } + /// Answers a read or write `.again` and hands back the ticket /// of the park, to answer it through `answerWith()` when the backend /// has something (lib9p's kept Req); null when it could not @@ -230,6 +257,7 @@ pub fn Runner(comptime Backend: type, comptime opts: fs.Options, comptime limits c.flags = .init(0); c.signal = .unset; c.room = .unset; + c.output_room = .init; // Under the lock: the slot is already visible as live. c.lock(); c.engine.initIn(.{ .in = &c.in, .out = &c.out, .root = r.root, .seed = r.seed ^ c.index }); @@ -315,6 +343,7 @@ pub fn Runner(comptime Backend: type, comptime opts: fs.Options, comptime limits return; }; c.engine.wrote(pending.len); + c.output_room.broadcast(io); const dead = c.engine.protocol.dead; greeted = greeted or c.engine.protocol.msize != 0; c.unlock(); diff --git a/test/web/catalog_ui.mjs b/test/web/catalog_ui.mjs index 9f4aca5..32fb2ad 100644 --- a/test/web/catalog_ui.mjs +++ b/test/web/catalog_ui.mjs @@ -47,6 +47,7 @@ async function readySites() { await wait(() => evaluate(`document.querySelectorA try { const root = path.join(work, 'public'); await fs.mkdir(root); await fs.writeFile(path.join(root, 'index.html'), '

Fixture

'); + await fs.writeFile(path.join(root, 'large.bin'), Buffer.alloc(512000, 0xa5)); await fs.writeFile(path.join(work, 'large.txt'), 'x'.repeat(180000)); const dashboard = await port(), debugPort = await port(); const runtime = path.join(work, 'runtime'); @@ -58,6 +59,9 @@ try { const web = startWeb(); const url = `http://127.0.0.1:${dashboard}`; await wait(async () => (await fetch(url)).ok, 'server ready'); + const concurrent = start(process.execPath, ['test/web/parallel_reads.mjs', url + '/sites/fixture/', root]); + const [parallelCode] = await once(concurrent, 'exit'); + assert.equal(parallelCode, 0, concurrent.output); assert.equal((await (await fetch(url + '/_cloud9/config.json')).json()).catalog, true); assert.equal((await fetch(url + '/_cloud9/catalog.js')).status, 404, 'standalone JS removed'); start(process.env.CLOUD9_CHROME || 'google-chrome-stable', ['--headless=new', '--no-sandbox', '--disable-dev-shm-usage', '--no-first-run', '--no-default-browser-check', `--user-data-dir=${work}/chrome`, `--remote-debugging-port=${debugPort}`, 'about:blank']); diff --git a/test/web/parallel_reads.mjs b/test/web/parallel_reads.mjs new file mode 100644 index 0000000..a95976b --- /dev/null +++ b/test/web/parallel_reads.mjs @@ -0,0 +1,44 @@ +// Read-only stress check: concurrent multi-frame website reads must be exact. +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; + +const [base, root] = process.argv.slice(2); +if (!base || !root) throw new Error('usage: node parallel_reads.mjs URL DIRECTORY'); +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const fileUrl = name => base.replace(/\/$/, '') + '/' + name.split(path.sep).map(encodeURIComponent).join('/'); +const files = []; +async function collect(dir, rel = '') { + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const name = path.join(rel, entry.name), full = path.join(dir, entry.name); + if (entry.isDirectory()) await collect(full, name); + else if (entry.isFile()) { + const bytes = await fs.readFile(full); + if (bytes.length >= 65536) files.push({ name, size: bytes.length, hash: hash(bytes) }); + } + } +} +await collect(root); +assert.ok(files.length > 0, 'need at least one multi-frame file'); +for (let round = 0; round < 5; round++) { + const reads = Array.from({ length: 16 }, async (_, i) => { + const file = files[(round * 16 + i) % files.length]; + const response = await fetch(fileUrl(file.name)); + assert.equal(response.status, 200, file.name); + const bytes = Buffer.from(await response.arrayBuffer()); + assert.equal(bytes.length, file.size, `truncated: ${file.name}`); + assert.equal(hash(bytes), file.hash, `corrupt: ${file.name}`); + }); + // Mix synchronous stats with reads, and abandon owned responses midway. + const canceled = Array.from({ length: 8 }, async (_, i) => { + const response = await fetch(fileUrl(files[i % files.length].name)); + assert.equal(response.status, 200); + const reader = response.body.getReader(); + await reader.read(); + await reader.cancel(); + }); + const head = fetch(fileUrl(files[0].name), { method: 'HEAD' }).then(r => assert.equal(r.status, 200)); + await Promise.all([...reads, ...canceled, head]); +} +console.log(`parallel_reads: 80 exact multi-frame responses, 16 concurrent, 40 cancellations and mixed HEAD (${files.length} source files)`); diff --git a/web/catalog.zig b/web/catalog.zig index d375973..977e0ae 100644 --- a/web/catalog.zig +++ b/web/catalog.zig @@ -540,13 +540,15 @@ pub const Catalog = struct { } fn readTask(cat: *Catalog, conn: *Runner.Conn, req: fs.Req, file: Io.File, slot: usize) void { defer file.close(cat.io); + defer { + cat.mutex.lockUncancelable(cat.io); + cat.reads[conn.index][slot] = 0; + cat.mutex.unlock(cat.io); + } var arena = std.heap.ArenaAllocator.init(A); defer arena.deinit(); const b = arena.allocator().alloc(u8, req.size) catch { - cat.mutex.lockUncancelable(cat.io); - defer cat.mutex.unlock(cat.io); - replyError(conn, req.tag, error.OutOfMemory); - cat.reads[conn.index][slot] = 0; + conn.replyAsync(&.{ .tag = req.tag, .status = .err, .ename = "OutOfMemory" }, "") catch {}; return; }; const result = file.readPositional(cat.io, &.{b}, req.off) catch |err| switch (err) { @@ -556,10 +558,7 @@ pub const Catalog = struct { }, else => err, }; - cat.mutex.lockUncancelable(cat.io); - defer cat.mutex.unlock(cat.io); - if (result) |n| conn.reply(&.{ .tag = req.tag }, b[0..n]) else |err| replyError(conn, req.tag, err); - cat.reads[conn.index][slot] = 0; + if (result) |n| conn.replyAsync(&.{ .tag = req.tag }, b[0..n]) catch {} else |err| conn.replyAsync(&.{ .tag = req.tag, .status = .err, .ename = @errorName(err) }, "") catch {}; } fn replyError(conn: *Runner.Conn, tag: u64, err: anyerror) void { const message = switch (err) { diff --git a/web/httpfs.zig b/web/httpfs.zig index 31c4014..479dd09 100644 --- a/web/httpfs.zig +++ b/web/httpfs.zig @@ -313,7 +313,9 @@ fn FS(comptime M: type) type { if (sent >= limit or data.len == 0) break; offset += data.len; const r = try self.readAt(leaf, offset, self.chunk()); - if (r.msg != .rread) break; + // Do not terminate chunked HTTP successfully after a 9P + // error: the client must detect the incomplete response. + if (r.msg != .rread) return error.Upstream; data = r.msg.rread.data; if (data.len == 0) break; } -- cgit v1.3