diff options
| -rw-r--r-- | build.zig | 3 | ||||
| -rw-r--r-- | docs/9web.service | 4 | ||||
| -rw-r--r-- | docs/catalog.md | 99 | ||||
| -rw-r--r-- | test/web/catalog_ui.mjs | 52 | ||||
| -rw-r--r-- | test/web/host_fs.py | 161 | ||||
| -rw-r--r-- | web/catalog.zig | 299 | ||||
| -rw-r--r-- | web/client.zig | 4 | ||||
| -rw-r--r-- | web/httpfs.zig | 45 | ||||
| -rw-r--r-- | web/static/app.mjs | 100 | ||||
| -rw-r--r-- | web/static/catalog.mjs | 6 | ||||
| -rw-r--r-- | web/static/client.mjs | 3 | ||||
| -rw-r--r-- | web/static/index.html | 4 | ||||
| -rw-r--r-- | web/static/style.css | 15 |
13 files changed, 660 insertions, 135 deletions
@@ -44,6 +44,9 @@ pub fn build(b: *std.Build) void { const catalog_test = b.addSystemCommand(&.{ "python3", "test/web/catalog.py" }); catalog_test.addArtifactArg(bridge); b.step("catalog-test", "Test local sites, controls, watcher and passive discovery").dependOn(&catalog_test.step); + const host_fs_test = b.addSystemCommand(&.{ "python3", "test/web/host_fs.py" }); + host_fs_test.addArtifactArg(bridge); + b.step("host-fs-test", "Test whole-host filesystem and 9P mount browsing").dependOn(&host_fs_test.step); const catalog_ui = b.addSystemCommand(&.{ "node", "test/web/catalog_ui.mjs" }); catalog_ui.addArtifactArg(bridge); b.step("catalog-ui-test", "Test integrated Files, Sites and Services views in Chromium").dependOn(&catalog_ui.step); diff --git a/docs/9web.service b/docs/9web.service index a20bd89..7109935 100644 --- a/docs/9web.service +++ b/docs/9web.service @@ -1,9 +1,9 @@ [Unit] -Description=9web local sites, build controls and listener catalogue +Description=9web filesystem, sites and services After=network.target [Service] -ExecStart=%h/.local/libexec/9web --listen [::]:8090 --catalog %h/.config/9web/sites.json --catalog-socket %t/9web/catalog.sock --state-dir %h/.local/state/9web +ExecStart=%h/.local/bin/9ns --mntgen -- %h/.local/libexec/9web --listen [::]:8090 --catalog %h/.config/9web/sites.json --catalog-socket %t/9web/catalog.sock --state-dir %h/.local/state/9web Restart=on-failure RestartSec=2 RuntimeDirectory=9web diff --git a/docs/catalog.md b/docs/catalog.md index ae6cd46..74062bb 100644 --- a/docs/catalog.md +++ b/docs/catalog.md @@ -6,6 +6,27 @@ and inventories listening TCP sockets without making requests to them. No separate site servers are needed. The existing `--upstream` gateway mode is unchanged; catalogue mode and an explicit upstream are mutually exclusive. +Set the top-level registry field `"filesystem": "/"` to expose the whole +workstation in the Files tab. The real `/` remains the root; the catalogue is +mounted at `/_9web`. Startup refuses an existing real `/_9web` rather than hiding +it. Without this opt-in field, the previous catalogue-only tree is unchanged. +Only `/` is accepted: this export is not a restricted directory sandbox. + +Host files support reading, editing, uploads, mkdir, rename and removal through +the normal 9P/WebSocket and HTTP `/fs` interfaces. Symlinks and mount points are +followed, including `/mnt/9p`. Access is checked by the kernel as the service's +user, not root; the 9P permission bits are permissive because its engine cannot +represent POSIX group/ACL checks. Device nodes, FIFOs and sockets are listed but +refused as file contents. Open files retain their descriptors across reads. +Blocking reads use bounded background tasks so one FUSE event stream does not +block all requests on the shared 9P connection. Non-seekable FUSE files use +streaming reads/writes rather than positional I/O. + +The installed service runs `9ns --mntgen -- 9web ...`: `/mnt/9p` is the posted +registry in that service's namespace, not an assumption about another shell's +mount namespace. No host-global mount is changed. Newly posted services are +discovered by 9ns in the usual way. + ## Configuration and serving See [catalog.example.json](catalog.example.json). Every site has a unique `name`, @@ -45,6 +66,12 @@ browser. The root opens files; `/?view=sites` and `/?view=services` are direct links to the other tabs. Browser history and reload preserve the selected view. Switching tabs preserves unsaved file edits. Ordinary upstream mode shows only files and does not poll the catalogue API. +Files uses the main directory table and breadcrumbs; there is no duplicate +sidebar tree. +Each file entry has a `follow` link (`?follow=1`) which opens metadata first and +starts an SSE read without first waiting for a finite snapshot. Use it for event +streams; stop does not reopen the file or discard received text. Ordinary file +navigation still reads finite contents for editing. The sites table has build, watch and cancel controls, plus files and log links that open the same file browser. Builds execute in the @@ -85,6 +112,11 @@ symlinks below that root are refused. The catalogue is itself a 9P backend, consumed by the ordinary gateway mux. `--serve` and WebSocket clients see the same tree as direct socket clients: +With `"filesystem": "/"`, prefix every catalogue path below with `/_9web`. +For example, the control URL becomes `/fs/_9web/sites/entomology/ctl` and its +mounted path is `/mnt/9p/web/_9web/sites/entomology/ctl`. UI links and controls +select the prefix automatically from the catalogue snapshot's `namespace`. + ``` /index.html /sites/<name>/public/ complete generated output tree (read-only) @@ -155,18 +187,35 @@ and the configured project build tools in PATH. Miniserve remains independent. This is a trusted-network operator interface: it has **no login or TLS**. Anyone who can reach the dashboard can read the catalogue and trigger configured jobs. +With `"filesystem": "/"`, they can additionally read or modify any file the +service user can access, including credentials, configuration and source code, +and operate control files under `/mnt/9p`. This can amount to arbitrary code +execution as that user. Only enable this export for clients you fully trust. Browser control requests reject foreign Origins; numeric Host validation and separate site origins prevent served pages from using the dashboard as a same-origin control API. This is not authentication. Bind to loopback or place an authenticated gateway in front of it when those network permissions are inappropriate. No -arbitrary command can be supplied through HTTP or 9P; commands come from the local -registry, which should remain writable only by its owner. +arbitrary command can be supplied to the catalogue build-control parser itself; +its commands come from the local registry. A whole-host writable export removes +that as a security boundary, since clients can edit the registry and build inputs. ## Verification +Mounted synthetic files commonly report length zero and are non-seekable. Their +normal EOF must become an empty 9P read, not `Rerror EndOfStream`. The browser +reads until EOF, so that error previously discarded otherwise valid contents. +Directory HTTP reads are now sized to the receiving buffer, and JSON/HTML +listings stream rather than overflowing a fixed-size buffer or hiding a read +failure behind an empty listing. +The browser's WASM adapter also releases its previous borrowed reply frame before +submitting another request. Without that release, a full-msize read could fail +with `InputFull` even though small files worked. + ``` zig build catalog-test http-fs-test mux-test -Doptimize=ReleaseSafe zig build catalog-ui-test -Doptimize=ReleaseSafe +zig build -Doptimize=ReleaseSafe # install 9ns and the 9proc-demo fixture too +zig build host-fs-test -Doptimize=ReleaseSafe ``` The catalogue integration test uses temporary real sites and processes. It checks @@ -183,6 +232,41 @@ inspection screenshots to `.zig-cache/catalog-ui/`. It uses the installed `google-chrome-stable` binary (or `CLOUD9_CHROME`), with no npm dependencies. It also checks that live HTTP ports stay exclusive and recent connections do not prevent an immediate restart of the dashboard or site listeners. +`host-fs-test` additionally checks a 1,500-entry directory, mounted 9P reads, +control writes, non-seekable EOF, blocking FUSE reads concurrent with other +requests, and 40 flush/clunk cycles without exhausting background readers. +The Chromium test navigates a real mounted fixture, reads a 180 KB file and +opens/stops a live event stream directly from its directory entry. + +## Publication and namespace policy + +There are three separate mechanisms, not one registration service: + +* 9P servers may use `serve.Runner.listenPosted` to publish a Unix listener at + `$XDG_RUNTIME_DIR/9p/<name>`. 9ns lists the registry and connects on a walk. + No request is made to 9web; publication lifetime belongs to the producer. +* Website roots and build recipes in this catalogue are startup configuration + in `sites.json`, not self-registration. The private catalogue socket itself + currently uses `listen`, not `listenPosted`. +* HTTP listener discovery observes `/proc` socket ownership and process metadata. + It neither registers services nor probes HTTP endpoints. + +The Plan 9 reference was checked against the local Principia Softwarica sources: +`ROOT/docs/man/3/srv`, `kernel/devices/sys/devsrv.c`, +`lib_networking/lib9p/post.c`, `ROOT/docs/man/4/mntgen`, and +`ROOT/docs/man/4/exportfs`. Plan 9's `/srv` stores references to already-open +channels; Unix listeners are a Linux adaptation, not identical semantics. +Original `mntgen` creates mountpoints on demand; 9ns's registry auto-connection +is additional policy. Explicit configuration is not itself contrary to Plan 9: +`ROOT/docs/man/6/namespace` specifies configured mount/bind/import composition. + +The coherent next design step, **not implemented here**, is to expose a pure +site/build-control 9P view and compose it into the consumer's namespace, with +HTTP serving those same paths. Shared supervision can remain in one process; +no extra daemon is required per static site. Do not post the current mixed +host/catalogue root back under its own `/mnt/9p`: that creates recursion and +possible self-dependent waits. Arbitrary-upstream composition and unifying +the separate disk-based static-site HTTP handler remain outstanding. ## Installed workstation configuration @@ -190,11 +274,12 @@ The application is named **9web** in its header, footer and page titles. The existing `/_cloud9/` asset/protocol URLs and library module names are retained for compatibility; they are not a different application name. -Current limitation: catalogue mode replaces the upstream with its synthetic -tree. The Files tab therefore does not expose the workstation filesystem or -compose an existing upstream with the catalogue. Ordinary `--upstream` mode -still works, but the installed catalogue service is not a filesystem-browser -replacement for miniserve. +The workstation registry enables `"filesystem": "/"`. Files opens the host +root, including the service's live `/mnt/9p` registry mount; the virtual catalogue +is under `/_9web`. The unit wraps the separately installed 9web binary with 9ns. +Ordinary `--upstream` mode is unchanged. Composition with an arbitrary explicit +upstream is still not implemented; this mode composes the host filesystem with +the catalogue, and existing 9P servers are available through `/mnt/9p`. The initial workstation setup uses dashboard port 8090 and site ports 8091–8095: Entomology lab, 0x4200.cafe, Standards, Pardes website, and the older café hub archive. diff --git a/test/web/catalog_ui.mjs b/test/web/catalog_ui.mjs index 6120a51..eed6c0a 100644 --- a/test/web/catalog_ui.mjs +++ b/test/web/catalog_ui.mjs @@ -11,8 +11,8 @@ const binary = path.resolve(process.argv[2] || 'zig-out/bin/9web'); const work = await fs.mkdtemp(path.join(os.tmpdir(), '9web-ui-')); const children = [], exceptions = [], pending = new Map(); let socket, serial = 0; -function start(command, args) { - const p = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] }); +function start(command, args, env = process.env) { + const p = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }); p.output = ''; p.stdout.on('data', b => { p.output += b; }); p.stderr.on('data', b => { p.output += b; }); p.on('error', e => { p.output += e.message; }); children.push(p); return p; } @@ -47,9 +47,15 @@ async function readySites() { await wait(() => evaluate(`document.querySelectorA try { const root = path.join(work, 'public'); await fs.mkdir(root); await fs.writeFile(path.join(root, 'index.html'), '<h1>Fixture</h1>'); + await fs.writeFile(path.join(work, 'large.txt'), 'x'.repeat(180000)); const dashboard = await port(), sitePort = await port(), debugPort = await port(); - await fs.writeFile(path.join(work, 'config.json'), JSON.stringify({ sites: [{ name: 'fixture', title: 'Fixture notes', cwd: work, root, port: sitePort, build: ['sh', '-c', 'echo integrated-build'], inputs: ['public/index.html'] }] })); - const web = start(binary, ['--listen', `127.0.0.1:${dashboard}`, '--catalog', path.join(work, 'config.json'), '--catalog-socket', path.join(work, 'catalog.sock'), '--state-dir', path.join(work, 'state')]); + const runtime = path.join(work, 'runtime'); + await fs.mkdir(path.join(runtime, '9p'), { recursive: true }); + start(path.resolve('zig-out/bin/9proc-demo'), ['--unix', path.join(runtime, '9p', 'fixture')]); + await wait(() => fs.stat(path.join(runtime, '9p', 'fixture')), '9P fixture posted'); + await fs.writeFile(path.join(work, 'config.json'), JSON.stringify({ filesystem: '/', sites: [{ name: 'fixture', title: 'Fixture notes', cwd: work, root, port: sitePort, build: ['sh', '-c', 'echo integrated-build'], inputs: ['public/index.html'] }] })); + const startWeb = () => start(path.resolve('zig-out/bin/9ns'), ['--mntgen', '--', binary, '--listen', `127.0.0.1:${dashboard}`, '--catalog', path.join(work, 'config.json'), '--catalog-socket', path.join(work, 'catalog.sock'), '--state-dir', path.join(work, 'state')], { ...process.env, XDG_RUNTIME_DIR: runtime }); + const web = startWeb(); const url = `http://127.0.0.1:${dashboard}`; await wait(async () => (await fetch(url)).ok, 'server ready'); assert.equal((await (await fetch(url + '/_cloud9/config.json')).json()).catalog, true); @@ -69,11 +75,35 @@ try { await cdp('Page.navigate', { url }); await wait(() => evaluate(`document.getElementById('status')?.textContent.includes('entries')`), 'files loaded'); assert.equal(await evaluate(`document.querySelectorAll('header').length`), 1); + assert.equal(await evaluate(`document.querySelector('#sidebar')`), null); assert.equal(await evaluate(`document.querySelector('.brand').textContent`), '9web'); assert.equal(await evaluate(`document.querySelector('footer').textContent`), 'served by 9web'); assert.equal(await evaluate(`document.title`), '/ — 9web'); + assert.equal(await evaluate(`Boolean(document.querySelector('#entries a[data-path="/etc"]'))`), true); assert.equal(await evaluate(`getComputedStyle(document.documentElement).backgroundColor`), 'rgb(255, 255, 255)'); assert.equal(await evaluate(`document.getElementById('view-files').getAttribute('aria-current')`), 'page'); + // Browse a real mounted 9P tree, including finite zero-stat-length files. + async function browse(path) { + await evaluate(`document.getElementById('path').value = ${JSON.stringify(path)}; document.getElementById('path-form').requestSubmit()`); + await wait(() => evaluate(`document.getElementById('status').textContent === 'File loaded.' || document.getElementById('status').textContent.includes('entries')`), 'mounted path loaded'); + } + await browse('/mnt/9p/fixture/build'); + assert.equal(await evaluate(`Boolean(document.querySelector('#entries a[data-path="/mnt/9p/fixture/build/zig_version"]'))`), true); + await browse('/mnt/9p/fixture/runtime/pid'); + assert.match(await evaluate(`document.getElementById('editor').value`), /^\d+/); + await browse('/mnt/9p/fixture/runtime/ctl'); + assert.equal(await evaluate(`document.getElementById('editor').value`), ''); + await browse(path.join(work, 'large.txt')); + assert.equal(await evaluate(`document.getElementById('editor').value.length`), 180000, 'full-msize browser reads across several frames'); + await browse('/_9web/sites/fixture'); + await click('#entries a[data-path="/_9web/sites/fixture/events"][data-follow]'); + await wait(() => evaluate(`document.getElementById('stream').textContent.includes('revision')`), 'direct follow of blocking events'); + assert.equal(await evaluate(`document.getElementById('go').disabled`), false, 'stream does not block navigation'); + await click('#follow'); + assert.equal(await evaluate(`document.getElementById('status').textContent`), 'Stream stopped.'); + await browse('/'); + await fs.mkdir('.zig-cache/catalog-ui', { recursive: true }); + await fs.writeFile('.zig-cache/catalog-ui/files.png', Buffer.from((await cdp('Page.captureScreenshot', { format: 'png' })).data, 'base64')); await view('sites'); await readySites(); assert.equal(await evaluate(`document.title`), 'sites — 9web'); assert.equal(await evaluate(`document.querySelector('#sites-entries .site-name').href`), `http://127.0.0.1:${sitePort}/`); @@ -130,8 +160,18 @@ try { assert.match(competitor.output, /address already in use/); web.kill('SIGTERM'); await once(web, 'exit'); await fs.unlink(path.join(work, 'catalog.sock')); - start(binary, ['--listen', `127.0.0.1:${dashboard}`, '--catalog', path.join(work, 'config.json'), '--catalog-socket', path.join(work, 'catalog.sock'), '--state-dir', path.join(work, 'state')]); + startWeb(); await wait(async () => (await fetch(url)).ok && (await fetch(`http://127.0.0.1:${sitePort}`)).ok, 'immediate restart with recent HTTP traffic'); + // Opt-in, read-only smoke navigation against an installed workstation. + if (process.env.NINE_WEB_LIVE) { + for (const pathname of ['/mnt/9p', '/mnt/9p/agents', '/mnt/9p/agents/codex/history', '/mnt/9p/zmx/daybreak/command', '/mnt/9p/zmx/daybreak/cwd', '/usr/bin']) { + await cdp('Page.navigate', { url: process.env.NINE_WEB_LIVE + pathname }); + await wait(() => evaluate(`document.getElementById('path')?.value === ${JSON.stringify(pathname)} && (document.getElementById('status').textContent === 'File loaded.' || document.getElementById('status').textContent.includes('entries'))`), 'live navigation: ' + pathname, 30000).catch(async error => { + throw new Error(error.message + ': ' + await evaluate(`document.getElementById('status')?.textContent`)); + }); + console.log('live browser navigation passed:', pathname); + } + } console.log('catalog_ui: shared shell/style, tab history, file/log navigation, build control, focus, dirty edits and mobile layout passed'); } catch (e) { for (const child of children) console.error(child.output.slice(-5000)); @@ -141,5 +181,5 @@ try { for (const p of children.reverse()) { if (p.exitCode === null) { p.kill('SIGTERM'); await Promise.race([once(p, 'exit'), new Promise(r => setTimeout(r, 3000))]); if (p.exitCode === null) p.kill('SIGKILL'); } } - await fs.rm(work, { recursive: true, force: true }); + await fs.rm(work, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } diff --git a/test/web/host_fs.py b/test/web/host_fs.py new file mode 100644 index 0000000..7a55eec --- /dev/null +++ b/test/web/host_fs.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Exercise host export; mutations are confined to our temporary directory.""" +from catalog import Nine, eventually, http, port, string +import json +import os +from pathlib import Path +import socket +import struct +import subprocess +import sys +import tempfile + + +def main(): + binary = str(Path(sys.argv[1]).resolve()) + ns = Path('zig-out/bin/9ns').resolve() + with tempfile.TemporaryDirectory(prefix='9web-host-') as tmp: + root = Path(tmp) + p = port() + public = root/'public' + public.mkdir() + (public/'index.html').write_text('site still works') + (root/'source').write_text('original data') + (root/'alias').symlink_to(root, target_is_directory=True) + (root/'dangling').symlink_to(root/'missing') + os.mkfifo(root/'fifo') + large = root/'large' + large.mkdir() + for i in range(1500): + (large/f'entry-{i:04}').touch() + config = {'filesystem': '/', 'sites': [{'name': 'fixture', 'title': 'Fixture', 'root': str(public), 'cwd': tmp, 'port': port(), 'build': ['sh', '-c', 'echo host-build']}]} + (root/'config.json').write_text(json.dumps(config)) + runtime = root/'runtime' + registry = runtime/'9p' + registry.mkdir(parents=True) + demo = subprocess.Popen([str(Path('zig-out/bin/9proc-demo').resolve()), '--unix', str(registry/'fixture')], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + eventually(lambda: (registry/'fixture').exists()) + event_config = {'sites': [{**config['sites'][0], 'port': port()}]} + (root/'events.json').write_text(json.dumps(event_config)) + event_server = subprocess.Popen([binary, '--listen', f'127.0.0.1:{port()}', '--catalog', str(root/'events.json'), '--catalog-socket', str(registry/'events'), '--state-dir', str(root/'events-state')], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + eventually(lambda: (registry/'events').exists()) + args = [binary, '--listen', f'127.0.0.1:{p}', '--catalog', str(root/'config.json'), '--catalog-socket', str(root/'cat.sock'), '--state-dir', str(root/'state')] + # Match the deployed wrapper. No host mount is changed. + if not ns.exists(): + raise RuntimeError('build 9ns first: zig build 9ns') + proc = subprocess.Popen([str(ns), '--mntgen', '--', *args], env={**os.environ, 'XDG_RUNTIME_DIR': str(runtime)}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + try: + eventually(lambda: http(p)[0] == 200) + entries = json.loads(http(p, '/fs/')[2]) + assert {'home', 'etc', 'usr', 'mnt', '_9web'} <= {x['name'] for x in entries}, entries + assert http(p, '/fs/etc/hostname')[2] == Path('/etc/hostname').read_bytes() + assert http(p, '/fs'+tmp+'/alias/source')[2] == b'original data' + assert 'dangling' in {x['name'] for x in json.loads(http(p, '/fs'+tmp)[2])} + assert http(p, '/fs'+tmp+'/fifo')[0] >= 400 + listing = http(p, '/fs'+tmp+'/large') + assert listing[0] == 200 and len(json.loads(listing[2])) == 1500 + assert http(p, '/fs/dev/null')[0] >= 400 + assert http(p, '/fs/mnt/9p')[0] == 200 + assert 'fixture' in {x['name'] for x in json.loads(http(p, '/fs/mnt/9p')[2])} + assert http(p, '/fs/mnt/9p/fixture/build/zig_version')[0] == 200 + assert b'0.17' in http(p, '/fs/mnt/9p/fixture/build/zig_version')[2] + mounted = '/fs/mnt/9p/fixture' + assert http(p, mounted+'/scratch/upload', 'PUT', 'via FUSE')[0] == 201 + assert http(p, mounted+'/scratch/upload')[2] == b'via FUSE' + control = http(p, mounted+'/runtime/ctl', 'PUT', 'echo mount-control') + assert control[0] == 200, control + assert b'mount-control' in http(p, mounted+'/runtime/ctl')[2] + finite = Nine(str(root/'cat.sock')) + finite.walk('mnt/9p/fixture/runtime/ctl') + finite.rpc(112, struct.pack('<IB', 2, 0)) + data = finite.rpc(116, struct.pack('<IQI', 2, 0, 4096))[4:] + assert b'mount-control' in data + assert finite.rpc(116, struct.pack('<IQI', 2, len(data), 4096))[4:] == b'' + finite.s.close() + assert http(p, mounted+'/scratch/upload', 'DELETE')[0] == 204 + # A blocking read through FUSE must not stall the shared backend. + events = Nine(str(root/'cat.sock')) + events.walk('mnt/9p/events/sites/fixture/events') + events.rpc(112, struct.pack('<IB', 2, 0)) + assert json.loads(events.rpc(116, struct.pack('<IQI', 2, 0, 4096))[4:])['phase'] == 'idle' + request = struct.pack('<IQI', 2, 0, 4096) + events.s.sendall(struct.pack('<IBH', len(request)+7, 116, 2) + request) + events.rpc(124, struct.pack('<I', 1), 3) + # Also wake/retry this parked request from an unrelated catalogue change. + assert http(p, '/fs/_9web/sites/fixture/ctl', 'PUT', 'build')[0] == 200 + assert http(p, '/fs/mnt/9p/events/sites/fixture/ctl', 'PUT', 'build')[0] == 200 + size, typ, tag = struct.unpack('<IBH', events.recv(7)) + assert (typ, tag) == (117, 2) + assert json.loads(events.recv(size-7)[4:])['revision'] > 1 + events.s.close() + eventually(lambda: json.loads(http(p, '/fs/mnt/9p/events/sites/fixture/status')[2])['generation'] == 1) + # Repeated flush+clunk must release the underlying blocking reads, + # rather than exhaust the bounded task table or steal later events. + cancel = Nine(str(root/'cat.sock')) + for _ in range(40): + cancel.walk('mnt/9p/events/sites/fixture/events') + cancel.rpc(112, struct.pack('<IB', 2, 0)) + cancel.rpc(116, struct.pack('<IQI', 2, 0, 4096)) + cancel.s.sendall(struct.pack('<IBH', len(request)+7, 116, 2) + request) + cancel.s.sendall(struct.pack('<IBHH', 9, 108, 3, 2)) + # An interrupted old read may be answered before Rflush. + while True: + n, kind, tag = struct.unpack('<IBH', cancel.recv(7)) + cancel.recv(n-7) + if (kind, tag) == (109, 3): + break + assert tag == 2 and kind in (107, 117), (kind, tag) + cancel.rpc(120, struct.pack('<I', 2), 4) + cancel.s.close() + # Directory and upload/edit/append/delete use the same HTTP/9P path. + target = '/fs'+tmp+'/created' + assert http(p, target+'/', 'PUT')[0] == 201 + assert http(p, target+'/file', 'PUT', 'abcdef')[0] == 201 + assert http(p, target+'/file', 'PUT', 'xy')[0] == 200 + assert (root/'created/file').read_bytes() == b'xy' + assert http(p, target+'/file?append=1', 'PUT', 'z')[0] == 200 + assert http(p, target+'/file')[2] == b'xyz' + assert http(p, target+'/file', headers={'Range': 'bytes=1-2'})[2] == b'yz' + assert http(p, target+'/file', 'HEAD')[1]['content-length'] == '3' + # The local namespace is also accessible through raw 9P. + nine = Nine(str(root/'cat.sock')) + rel = tmp.lstrip('/') + assert nine.read(rel+'/source') == b'original data' + nine.write(rel+'/created/file', 'short') + assert (root/'created/file').read_text() == 'short' + # Twstat rename on the backing filesystem (not a UI copy/delete). + nine.walk(rel+'/created/file') + stat = struct.pack('<HI', 0xffff, 0xffffffff) + b'\xff'*13 + struct.pack('<IIIQ', 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffffffffffff) + string('renamed') + string('')*3 + stat = struct.pack('<H', len(stat)) + stat + nine.rpc(126, struct.pack('<IH', 2, len(stat)) + stat) + nine.rpc(120, struct.pack('<I', 2)) + assert (root/'created/renamed').read_text() == 'short' + # Virtual tree stays read-only apart from ctl. + assert http(p, '/fs/_9web/new', 'PUT', 'bad')[0] == 403 + assert http(p, '/fs/_9web/sites/fixture/public/index.html', 'PUT', 'bad')[0] == 403 + assert http(p, '/fs/_9web/sites/fixture/ctl', 'PUT', 'build')[0] == 200 + eventually(lambda: b'host-build' in http(p, '/fs/_9web/sites/fixture/log')[2]) + assert http(config['sites'][0]['port'])[2] == b'site still works' + assert http(p, '/_cloud9/catalog.json')[0] == 200 + assert json.loads(http(p, '/_cloud9/catalog.json')[2])['namespace'] == '/_9web' + assert http(p, target+'/renamed', 'DELETE')[0] == 204 + assert http(p, target, 'DELETE')[0] == 204 + nine.s.close() + print('host_fs: root, symlinks, special-file refusal, /mnt/9p, upload/edit/append/rename/delete, raw 9P and catalogue controls passed') + finally: + proc.terminate() + try: + output = proc.communicate(timeout=5)[0] + except subprocess.TimeoutExpired: + proc.kill() + output = proc.communicate()[0] + if sys.exc_info()[0]: + print(output.decode(errors='replace'), file=sys.stderr) + demo.terminate() + demo.wait(timeout=5) + event_server.terminate() + event_server.wait(timeout=5) + + +if __name__ == '__main__': + main() diff --git a/web/catalog.zig b/web/catalog.zig index 53bab19..87349bb 100644 --- a/web/catalog.zig +++ b/web/catalog.zig @@ -8,6 +8,7 @@ const fs = c9.fs; const Backend = struct { pub const Req = fs.Req; pub const Reply = fs.Reply; + pub const features: fs.Features = .{ .create = true, .remove = true, .wstat = true }; }; pub const Runner = c9.serve.Runner(Backend, .{ .fid_capacity = 512, .name_capacity = 255 }, .{ .msize = 65536, .connections = 16 }); @@ -22,7 +23,7 @@ pub const Site = struct { // Poll only these source paths for a managed rebuild watcher. inputs: []const []const u8 = &.{}, }; -const Settings = struct { sites: []const Site }; +const Settings = struct { sites: []const Site, filesystem: ?[]const u8 = null }; const State = struct { phase: []const u8 = "idle", watching: bool = false, @@ -55,12 +56,24 @@ pub const Catalog = struct { group: Io.Group = .init, runner: *Runner, http_port: u16 = 0, + host_filesystem: bool = false, + files: [16][512]?Io.File = @splat(@splat(null)), + reads: [16][32]u64 = @splat(@splat(0)), + read_handles: [16][32]u32 = @splat(@splat(0)), + read_groups: [16][32]Io.Group = @splat(@splat(.init)), cursors: [16][64]EventCursor = @splat(@splat(.{})), pub fn init(io: Io, config: []const u8, state_dir: []const u8) !*Catalog { const bytes = try Io.Dir.cwd().readFileAlloc(io, config, A, .limited(1024 * 1024)); defer A.free(bytes); const parsed = try std.json.parseFromSlice(Settings, A, bytes, .{ .allocate = .alloc_always }); + if (parsed.value.filesystem) |root| { + // This is intentionally a whole-host export, not a sandbox/chroot. + if (!std.mem.eql(u8, root, "/")) return error.FilesystemMustBeRoot; + if (Io.Dir.cwd().statFile(io, "/_9web", .{ .follow_symlinks = false })) |_| { + return error.CatalogMountCollision; + } else |err| if (err != error.FileNotFound) return err; + } // Config is immutable for the lifetime of the process. var names: std.StringHashMap(void) = .init(A); defer names.deinit(); @@ -75,9 +88,10 @@ pub const Catalog = struct { try Io.Dir.cwd().createDirPath(io, state_dir); const cat = try A.create(Catalog); cat.* = .{ .io = io, .sites = parsed.value.sites, .states = try A.alloc(State, parsed.value.sites.len), .state_dir = state_dir, .runner = try A.create(Runner) }; + cat.host_filesystem = parsed.value.filesystem != null; @memset(cat.states, .{}); try cat.nodes.append(A, ""); - cat.runner.init(.{ .io = io, .root = 1, .handler = .{ .ctx = cat, .serve = serve } }); + cat.runner.init(.{ .io = io, .root = 1, .handler = .{ .ctx = cat, .serve = serve, .closed = closed } }); return cat; } @@ -130,7 +144,7 @@ pub const Catalog = struct { const rows = try a.alloc(Row, cat.sites.len); for (cat.sites, 0..) |s, i| rows[i] = .{ .name = s.name, .title = s.title, .root = s.root, .port = s.port, .build = s.build.len != 0, .watch = s.watch.len != 0 or s.inputs.len != 0, .status = cat.states[i] }; const listeners = try std.json.parseFromSlice(std.json.Value, a, cat.listeners, .{}); - return std.json.Stringify.valueAlloc(a, .{ .sites = rows, .listeners = listeners.value, .discovery_error = cat.discovery_error }, .{}); + return std.json.Stringify.valueAlloc(a, .{ .sites = rows, .listeners = listeners.value, .discovery_error = cat.discovery_error, .namespace = if (cat.host_filesystem) "/_9web" else "" }, .{}); } fn discoverLoop(cat: *Catalog) void { @@ -304,7 +318,14 @@ pub const Catalog = struct { try cat.nodes.append(A, try A.dupe(u8, path)); return cat.nodes.items.len; } - fn sitePath(cat: *Catalog, path: []const u8) ?struct { i: usize, rest: []const u8 } { + fn virtualPath(cat: *Catalog, path: []const u8) ?[]const u8 { + if (!cat.host_filesystem) return path; + if (std.mem.eql(u8, path, "_9web")) return ""; + if (std.mem.startsWith(u8, path, "_9web/")) return path[6..]; + return null; + } + fn sitePath(cat: *Catalog, raw: []const u8) ?struct { i: usize, rest: []const u8 } { + const path = cat.virtualPath(raw) orelse return null; if (!std.mem.startsWith(u8, path, "sites/")) return null; const rest = path[6..]; const n = std.mem.indexOfScalar(u8, rest, '/') orelse rest.len; @@ -316,14 +337,15 @@ pub const Catalog = struct { result.node = try cat.intern(path); return result; } - fn attrData(cat: *Catalog, a: std.mem.Allocator, path: []const u8) !fs.Attr { - var result: fs.Attr = .{ .name = std.fs.path.basename(path), .mode = 0o444 }; + fn attrData(cat: *Catalog, a: std.mem.Allocator, raw: []const u8) !fs.Attr { + const path = cat.virtualPath(raw) orelse return cat.hostAttr(a, raw); + var result: fs.Attr = .{ .name = std.fs.path.basename(raw), .mode = 0o444 }; if (path.len == 0 or std.mem.eql(u8, path, "sites") or std.mem.eql(u8, path, "services") or std.mem.eql(u8, path, "listeners")) { result.dir = true; result.mode = 0o555; return result; } - if (cat.sitePath(path)) |p| { + if (cat.sitePath(raw)) |p| { if (p.rest.len == 0 or std.mem.eql(u8, p.rest, "public")) { result.dir = true; result.mode = 0o555; @@ -344,11 +366,12 @@ pub const Catalog = struct { return result; } } - const data = try cat.content(a, path); + const data = try cat.content(a, raw); result.size = data.len; return result; } - fn content(cat: *Catalog, a: std.mem.Allocator, path: []const u8) ![]const u8 { + fn content(cat: *Catalog, a: std.mem.Allocator, raw: []const u8) ![]const u8 { + const path = cat.virtualPath(raw) orelse return error.FileNotFound; if (std.mem.eql(u8, path, "index.html")) return @embedFile("static/index.html"); if (std.mem.eql(u8, path, "listeners/index.json")) return cat.listeners; if (std.mem.eql(u8, path, "services/index.json")) { @@ -359,7 +382,7 @@ pub const Catalog = struct { }; return std.json.Stringify.valueAlloc(a, known.items, .{}); } - if (cat.sitePath(path)) |p| { + if (cat.sitePath(raw)) |p| { if (std.mem.eql(u8, p.rest, "status") or std.mem.eql(u8, p.rest, "events")) return std.json.Stringify.valueAlloc(a, cat.states[p.i], .{}); if (std.mem.eql(u8, p.rest, "ctl")) return "build\nwatch on\nwatch off\ncancel\n"; if (std.mem.eql(u8, p.rest, "log")) { @@ -373,11 +396,20 @@ pub const Catalog = struct { } return error.FileNotFound; } - fn children(cat: *Catalog, a: std.mem.Allocator, path: []const u8) ![][]const u8 { + fn children(cat: *Catalog, a: std.mem.Allocator, raw: []const u8) ![][]const u8 { var names: std.ArrayList([]const u8) = .empty; + const path = cat.virtualPath(raw) orelse { + var dir = try Io.Dir.cwd().openDir(cat.io, try hostPath(a, raw), .{ .iterate = true }); + defer dir.close(cat.io); + var it = dir.iterate(); + while (try it.next(cat.io)) |entry| try names.append(a, try a.dupe(u8, entry.name)); + if (raw.len == 0) try names.append(a, "_9web"); + sortNames(names.items); + return names.items; + }; if (path.len == 0) try names.appendSlice(a, &.{ "index.html", "sites", "services", "listeners" }) else if (std.mem.eql(u8, path, "sites")) { for (cat.sites) |s| try names.append(a, s.name); - } else if (std.mem.eql(u8, path, "services") or std.mem.eql(u8, path, "listeners")) try names.append(a, "index.json") else if (cat.sitePath(path)) |p| { + } else if (std.mem.eql(u8, path, "services") or std.mem.eql(u8, path, "listeners")) try names.append(a, "index.json") else if (cat.sitePath(raw)) |p| { if (p.rest.len == 0) try names.appendSlice(a, &.{ "public", "ctl", "status", "log", "events" }) else if (std.mem.eql(u8, p.rest, "public") or std.mem.startsWith(u8, p.rest, "public/")) { const rel = if (p.rest.len == 6) "" else p.rest[7..]; const f = try openBeneath(cat.io, cat.sites[p.i].root, rel); @@ -389,17 +421,46 @@ pub const Catalog = struct { }; } else return error.NotDir; } else return error.NotDir; - std.mem.sort([]const u8, names.items, {}, struct { + sortNames(names.items); + return names.items; + } + fn sortNames(names: [][]const u8) void { + std.mem.sort([]const u8, names, {}, struct { fn less(_: void, x: []const u8, y: []const u8) bool { return std.mem.lessThan(u8, x, y); } }.less); - return names.items; } fn serve(ctx: ?*anyopaque, conn: *Runner.Conn, req: fs.Req) void { const cat: *Catalog = @ptrCast(@alignCast(ctx.?)); var arena = std.heap.ArenaAllocator.init(A); defer arena.deinit(); + if (req.op == .release and req.opened and req.handle != 0) { + cat.mutex.lockUncancelable(cat.io); + const host = req.node > 0 and req.node <= cat.nodes.items.len and cat.virtualPath(cat.nodes.items[@intCast(req.node - 1)]) == null; + var cancel: [32]bool = @splat(false); + if (host) for (cat.reads[conn.index], 0..) |tag, ri| { + cancel[ri] = tag != 0 and cat.read_handles[conn.index][ri] == req.handle; + }; + cat.mutex.unlock(cat.io); + // Flush followed by clunk must interrupt kernel/FUSE reads too, + // not just discard the eventual 9P reply and keep consuming events. + for (cancel, 0..) |yes, ri| if (yes) cat.read_groups[conn.index][ri].cancel(cat.io); + } + // Files may be blocking 9P event streams through FUSE. Never hold the + // catalogue lock while reading/writing them; other connections must run. + if (req.op == .read or req.op == .write) { + cat.mutex.lockUncancelable(cat.io); + const host = req.node > 0 and req.node <= cat.nodes.items.len and cat.virtualPath(cat.nodes.items[@intCast(req.node - 1)]) == null; + const file = if (host and req.handle > 0 and req.handle <= 512) cat.files[conn.index][req.handle - 1] else null; + cat.mutex.unlock(cat.io); + if (host) { + if (req.op == .read) { + cat.startRead(conn, req, file) catch |err| replyError(conn, req.tag, err); + } else cat.hostIo(arena.allocator(), conn, req, file) catch |err| replyError(conn, req.tag, err); + return; + } + } // Controls take their own lock; all other node operations are serialized. if (req.op == .write) { cat.mutex.lockUncancelable(cat.io); @@ -421,13 +482,217 @@ pub const Catalog = struct { } cat.mutex.lockUncancelable(cat.io); defer cat.mutex.unlock(cat.io); - cat.answer(arena.allocator(), conn, req) catch |err| conn.reply(&.{ .tag = req.tag, .status = .err, .ename = if (err == error.FileNotFound) "not found" else @errorName(err) }, ""); + cat.answer(arena.allocator(), conn, req) catch |err| replyError(conn, req.tag, err); + } + fn closed(ctx: ?*anyopaque, conn: *Runner.Conn) void { + const cat: *Catalog = @ptrCast(@alignCast(ctx.?)); + for (&cat.read_groups[conn.index]) |*group| group.cancel(cat.io); + cat.read_groups[conn.index] = @splat(.init); + cat.mutex.lockUncancelable(cat.io); + defer cat.mutex.unlock(cat.io); + cat.reads[conn.index] = @splat(0); + } + fn startRead(cat: *Catalog, conn: *Runner.Conn, req: fs.Req, file: ?Io.File) !void { + const f = file orelse return error.FileNotOpen; + cat.mutex.lockUncancelable(cat.io); + defer cat.mutex.unlock(cat.io); + // Park before dispatch, including retries caused by catalogue changes. + // A completed/flushed request has no ticket and must not be reissued. + _ = conn.hold(&.{ .tag = req.tag, .status = .again }) orelse return; + for (cat.reads[conn.index]) |tag| if (tag == req.tag) return; + var slot: usize = 0; + while (slot < 32 and cat.reads[conn.index][slot] != 0) : (slot += 1) {} + if (slot == 32) return error.TooManyPendingReads; + try cat.read_groups[conn.index][slot].await(cat.io); + cat.read_groups[conn.index][slot] = .init; + // Clunk/flush can release the original handle while this read waits. + const linux = std.os.linux; + const rc = linux.fcntl(f.handle, linux.F.DUPFD_CLOEXEC, 0); + if (linux.errno(rc) != .SUCCESS) return error.TooManyOpenFiles; + const duplicate: Io.File = .{ .handle = @intCast(rc), .flags = f.flags }; + errdefer duplicate.close(cat.io); + cat.reads[conn.index][slot] = req.tag; + cat.read_handles[conn.index][slot] = req.handle; + errdefer cat.reads[conn.index][slot] = 0; + try cat.read_groups[conn.index][slot].concurrent(cat.io, readTask, .{ cat, conn, req, duplicate, slot }); + } + fn readTask(cat: *Catalog, conn: *Runner.Conn, req: fs.Req, file: Io.File, slot: usize) void { + defer file.close(cat.io); + var arena = std.heap.ArenaAllocator.init(A); + defer arena.deinit(); + const b = arena.allocator().alloc(u8, req.size) catch { + cat.mutex.lockUncancelable(cat.io); + defer cat.mutex.unlock(cat.io); + replyError(conn, req.tag, error.OutOfMemory); + cat.reads[conn.index][slot] = 0; + return; + }; + const result = file.readPositional(cat.io, &.{b}, req.off) catch |err| switch (err) { + error.Unseekable => file.readStreaming(cat.io, &.{b}) catch |stream_err| switch (stream_err) { + error.EndOfStream => 0, + else => stream_err, + }, + else => err, + }; + cat.mutex.lockUncancelable(cat.io); + defer cat.mutex.unlock(cat.io); + if (result) |n| conn.reply(&.{ .tag = req.tag }, b[0..n]) else |err| replyError(conn, req.tag, err); + cat.reads[conn.index][slot] = 0; + } + fn replyError(conn: *Runner.Conn, tag: u64, err: anyerror) void { + const message = switch (err) { + error.FileNotFound => "not found", + error.AccessDenied, error.PermissionDenied => "permission denied", + error.PathAlreadyExists => "file exists", + else => @errorName(err), + }; + conn.reply(&.{ .tag = tag, .status = .err, .ename = message }, ""); + } + fn hostPath(a: std.mem.Allocator, path: []const u8) ![]const u8 { + if (!safeRelative(path)) return error.AccessDenied; + return std.fmt.allocPrint(a, "/{s}", .{path}); + } + fn hostAttr(cat: *Catalog, a: std.mem.Allocator, path: []const u8) !fs.Attr { + const full = try hostPath(a, path); + const st = Io.Dir.cwd().statFile(cat.io, full, .{}) catch |err| blk: { + // Keep dangling symlinks visible; an attempt to open reports the error. + if (err != error.FileNotFound) return err; + break :blk try Io.Dir.cwd().statFile(cat.io, full, .{ .follow_symlinks = false }); + }; + return .{ + .name = std.fs.path.basename(path), + .dir = st.kind == .directory, + .size = st.size, + .mtime = @intCast(@min(std.math.maxInt(u32), @max(0, st.mtime.toSeconds()))), + // The 9P engine has no POSIX uid/group/ACL context. The kernel is + // the authority for this export, operating as the service user. + .mode = if (st.kind == .directory) 0o777 else 0o666, + }; + } + fn hostOpen(cat: *Catalog, a: std.mem.Allocator, path: []const u8, mode: u8, truncate: bool) !Io.File { + // Pin the inode with O_PATH, inspect it without opening a device/FIFO, + // then reopen that exact inode. Symlinks and mount points are followed. + const ref = try Io.Dir.cwd().openFile(cat.io, try hostPath(a, path), .{ .path_only = true }); + defer ref.close(cat.io); + const st = try ref.stat(cat.io); + if (st.kind != .file and st.kind != .directory) return error.SpecialFileNotSupported; + const fd_path = try std.fmt.allocPrintSentinel(a, "/proc/self/fd/{d}", .{ref.handle}, 0); + const linux = std.os.linux; + const flags: linux.O = .{ .ACCMODE = switch (mode & 3) { + 1 => .WRONLY, + 2 => .RDWR, + else => .RDONLY, + }, .CLOEXEC = true, .NONBLOCK = true, .NOCTTY = true, .TRUNC = truncate }; + const rc = linux.open(fd_path, flags, 0); + switch (linux.errno(rc)) { + .SUCCESS => return .{ .handle = @intCast(rc), .flags = .{ .nonblocking = true } }, + .ACCES, .PERM => return error.AccessDenied, + .NOENT => return error.FileNotFound, + .ISDIR => return error.IsDir, + .ROFS => return error.PermissionDenied, + else => return error.HostOpenFailed, + } + } + fn hostIo(cat: *Catalog, a: std.mem.Allocator, conn: *Runner.Conn, req: fs.Req, file: ?Io.File) !void { + const f = file orelse return error.FileNotOpen; + if (req.op == .write) { + f.writePositionalAll(cat.io, req.data, req.off) catch |err| switch (err) { + error.Unseekable => try f.writeStreamingAll(cat.io, req.data), + else => return err, + }; + conn.reply(&.{ .tag = req.tag, .written = @intCast(req.data.len) }, ""); + } else { + const b = try a.alloc(u8, req.size); + // A short read is meaningful for synthetic files and event streams. + const n = try f.readPositional(cat.io, &.{b}, req.off); + conn.reply(&.{ .tag = req.tag }, b[0..n]); + } + } + fn hostOperation(cat: *Catalog, a: std.mem.Allocator, conn: *Runner.Conn, req: fs.Req, path: []const u8, reply: *fs.Reply) !void { + const full = try hostPath(a, path); + switch (req.op) { + .open => { + var slot: usize = 0; + while (slot < 512 and cat.files[conn.index][slot] != null) : (slot += 1) {} + if (slot == 512) return error.TooManyOpenFiles; + var next = path; + const f = if (req.create) blk: { + if (!safeName(req.data)) return error.AccessDenied; + next = try std.fs.path.join(a, &.{ path, req.data }); + if (cat.virtualPath(next) != null) return error.PermissionDenied; + const dest = try hostPath(a, next); + if (req.perm & c9.dmdir != 0) { + try Io.Dir.cwd().createDir(cat.io, dest, .fromMode(req.perm & 0o777)); + break :blk try cat.hostOpen(a, next, req.omode, false); + } + break :blk try Io.Dir.cwd().createFile(cat.io, dest, .{ .read = (req.omode & 3) != 1, .exclusive = true, .permissions = .fromMode(req.perm & 0o777) }); + } else try cat.hostOpen(a, path, req.omode, false); + errdefer f.close(cat.io); + reply.attr = try cat.attr(a, next); + reply.handle = @intCast(slot + 1); + cat.files[conn.index][slot] = f; + }, + .release => { + if (req.opened and req.handle > 0 and req.handle <= 512) { + if (cat.files[conn.index][req.handle - 1]) |f| f.close(cat.io); + cat.files[conn.index][req.handle - 1] = null; + } + if (req.remove) { + if (path.len == 0) return error.PermissionDenied; + const st = try Io.Dir.cwd().statFile(cat.io, full, .{ .follow_symlinks = false }); + if (st.kind == .directory) try Io.Dir.cwd().deleteDir(cat.io, full) else try Io.Dir.cwd().deleteFile(cat.io, full); + } + }, + .setattr => { + if (req.set.name and (req.set.mode or req.set.mtime or req.set.length)) return error.CombinedRenameNotSupported; + if (req.set.name) { + if (path.len == 0 or !safeName(req.data)) return error.PermissionDenied; + const next = try std.fs.path.join(a, &.{ std.fs.path.dirname(path) orelse "", req.data }); + if (cat.virtualPath(next) != null) return error.PermissionDenied; + // Allocate replacement names before changing anything on disk. + var replacements: std.ArrayList(struct { index: usize, name: []const u8 }) = .empty; + for (cat.nodes.items, 0..) |n, ni| { + if (std.mem.eql(u8, n, path) or (std.mem.startsWith(u8, n, path) and n.len > path.len and n[path.len] == '/')) + try replacements.append(a, .{ .index = ni, .name = try std.fmt.allocPrint(A, "{s}{s}", .{ next, n[path.len..] }) }); + } + var renamed = false; + errdefer if (!renamed) { + for (replacements.items) |r| A.free(r.name); + }; + try Io.Dir.cwd().renamePreserve(full, .cwd(), try hostPath(a, next), cat.io); + for (replacements.items) |r| { + A.free(cat.nodes.items[r.index]); + cat.nodes.items[r.index] = r.name; + } + renamed = true; + reply.attr = try cat.attr(a, next); + } else { + // OTRUNC must reach FUSE as an open flag, not ftruncate; + // synthetic 9P ctl files often do not implement setattr. + const f = try cat.hostOpen(a, path, if (req.truncate or req.set.length) 1 else 0, req.truncate); + defer f.close(cat.io); + if (req.set.length and !req.truncate) try f.setLength(cat.io, req.length); + if (req.set.mode) try f.setPermissions(cat.io, .fromMode(req.perm & 0o777)); + if (req.set.mtime) try f.setTimestamps(cat.io, .{ .modify_timestamp = .{ .new = .{ .nanoseconds = @as(i96, req.mtime) * std.time.ns_per_s } } }); + reply.attr = try cat.attr(a, path); + } + }, + else => unreachable, + } + } + fn safeName(name: []const u8) bool { + return name.len > 0 and safeRelative(name) and std.mem.indexOfScalar(u8, name, '/') == null; } fn answer(cat: *Catalog, a: std.mem.Allocator, conn: *Runner.Conn, req: fs.Req) !void { if (req.node == 0 or req.node > cat.nodes.items.len) return error.FileNotFound; const path = cat.nodes.items[@intCast(req.node - 1)]; var reply: fs.Reply = .{ .tag = req.tag }; var bytes: []const u8 = ""; + if (cat.virtualPath(path) == null and (req.op == .open or req.op == .release or req.op == .setattr)) { + try cat.hostOperation(a, conn, req, path, &reply); + conn.reply(&reply, ""); + return; + } switch (req.op) { .lookup => { const next = if (std.mem.eql(u8, req.data, ".")) path else if (std.mem.eql(u8, req.data, "..")) (std.fs.path.dirname(path) orelse "") else try std.fs.path.join(a, &.{ path, req.data }); @@ -435,10 +700,11 @@ pub const Catalog = struct { }, .getattr => reply.attr = try cat.attr(a, path), .setattr => { - if (!std.mem.endsWith(u8, path, "/ctl")) return error.PermissionDenied; + if (!std.mem.endsWith(u8, path, "/ctl") or req.set.name or req.set.mode or req.set.mtime or (req.set.length and req.length != 0)) return error.PermissionDenied; reply.attr = try cat.attr(a, path); }, .open => { + if (req.create) return error.PermissionDenied; if ((req.omode & 3) != 0 and !std.mem.endsWith(u8, path, "/ctl")) return error.PermissionDenied; if (std.mem.endsWith(u8, path, "/events")) { var found = false; @@ -452,6 +718,7 @@ pub const Catalog = struct { } }, .release => { + if (req.remove) return error.PermissionDenied; if (req.opened and req.handle != 0) cat.cursors[conn.index][req.handle - 1].used = false; }, .write => unreachable, diff --git a/web/client.zig b/web/client.zig index 2ec9163..54b010b 100644 --- a/web/client.zig +++ b/web/client.zig @@ -60,6 +60,10 @@ fn fail(message: []const u8) i32 { return -1; } fn submit(request: c9.Client.Request) i32 { + // take() lends reply bytes until its next call. JS has copied the result + // before submitting another request; release that frame now, before a + // full-msize reply arrives. Otherwise push() has less than msize free. + if (client.take() != null) return fail("UnexpectedBufferedReply"); _ = client.submit(request) catch |err| return fail(@errorName(err)); return 0; } diff --git a/web/httpfs.zig b/web/httpfs.zig index fa783e5..8af5793 100644 --- a/web/httpfs.zig +++ b/web/httpfs.zig @@ -294,16 +294,35 @@ fn FS(comptime M: type) type { fn listDir(self: *Self, st: wire.Stat) !void { const html = self.wantsHtml(); + var name_buf: [256]u8 = undefined; + const name = name_buf[0..@min(st.name.len, name_buf.len)]; + @memcpy(name, st.name[0..name.len]); const leaf = self.leaf_fid.?; const opened = try self.rpcSmall(.{ .open = .{ .fid = leaf, .mode = c9.oread } }); if (opened.msg == .rerror) return self.rerr(opened.msg.rerror.ename); if (opened.msg != .ropen) return error.Upstream; - // JSON/HTML is built into `big`; directory data is read into `small`. - var w: std.Io.Writer = .fixed(self.big); + // Read frames must fit the RPC scratch buffer. Stream the listing + // so large directories neither overflow `big` nor appear empty. + const count: u32 = @intCast(@min(self.chunk(), self.small.len - 11)); + var r: M.Rpc = .{ .buf = self.small }; + var got = try self.m.rpc(.{ .read = .{ .fid = leaf, .offset = 0, .count = count } }, &r); + if (got.msg == .rerror) return self.rerr(got.msg.rerror.ename); + if (got.msg != .rread) return error.Upstream; + self.responded = true; + var bw = try self.request.respondStreaming(self.big, .{ .respond_options = .{ + .status = .ok, + .keep_alive = false, + .extra_headers = &.{ + .{ .name = "content-type", .value = if (html) "text/html; charset=utf-8" else "application/json" }, + .{ .name = "cache-control", .value = "no-store" }, + .{ .name = "x-content-type-options", .value = "nosniff" }, + }, + } }); + const w = &bw.writer; if (html) { w.writeAll("<!doctype html><meta charset=utf-8><title>") catch return error.Upstream; - writeHtml(&w, st.name); + writeHtml(w, name); w.writeAll("</title><ul>") catch return error.Upstream; } else { w.writeByte('[') catch return error.Upstream; @@ -311,37 +330,31 @@ fn FS(comptime M: type) type { var offset: u64 = 0; var first = true; while (true) { - var r: M.Rpc = .{ .buf = self.small }; - const got = try self.m.rpc(.{ .read = .{ .fid = leaf, .offset = offset, .count = self.chunk() } }, &r); - if (got.msg != .rread) break; const data = got.msg.rread.data; if (data.len == 0) break; offset += data.len; var rest: []const u8 = data; while (rest.len >= 2) { const size: usize = @as(usize, std.mem.readInt(u16, rest[0..2], .little)) + 2; - if (size > rest.len) break; - const entry = c9.Stat.decode(rest[0..size]) catch break; + if (size > rest.len) return error.Upstream; + const entry = c9.Stat.decode(rest[0..size]) catch return error.Upstream; rest = rest[size..]; if (html) { w.writeAll("<li>") catch return error.Upstream; - writeHtml(&w, entry.name); + writeHtml(w, entry.name); if (entry.qid.type & c9.qtdir != 0) w.writeByte('/') catch {}; w.writeAll("</li>") catch return error.Upstream; } else { if (!first) w.writeByte(',') catch return error.Upstream; first = false; - writeEntryJson(&w, entry) catch return error.Upstream; + writeEntryJson(w, entry) catch return error.Upstream; } } + got = try self.m.rpc(.{ .read = .{ .fid = leaf, .offset = offset, .count = count } }, &r); + if (got.msg != .rread) return error.Upstream; } if (html) w.writeAll("</ul>") catch return error.Upstream else w.writeByte(']') catch return error.Upstream; - self.responded = true; - self.request.respond(w.buffered(), .{ .status = .ok, .keep_alive = false, .extra_headers = &.{ - .{ .name = "content-type", .value = if (html) "text/html; charset=utf-8" else "application/json" }, - .{ .name = "cache-control", .value = "no-store" }, - .{ .name = "x-content-type-options", .value = "nosniff" }, - } }) catch {}; + try bw.end(); } // -- writes -------------------------------------------------------- diff --git a/web/static/app.mjs b/web/static/app.mjs index f18a103..33d5a08 100644 --- a/web/static/app.mjs +++ b/web/static/app.mjs @@ -21,7 +21,7 @@ function locationView() { return configuration?.catalog && ['sites', 'services'].includes(view) ? view : 'files'; } function viewURL(view, path = currentPath) { - return pathURL(path) + (view === 'files' ? '' : `?view=${view}`); + return pathURL(path) + (view === 'files' ? (current?.stream && path === currentPath ? '?follow=1' : '') : `?view=${view}`); } function showView(view) { activeView = view; @@ -43,7 +43,7 @@ function status(message, error = false) { function controls() { for (const id of ['go', 'reload', 'newfile', 'newdir', 'upload']) $(id).disabled = busy; $('save').disabled = busy || !current || current.stat.directory || $('editor').hidden || !dirty; - $('download').disabled = busy || !current || current.stat.directory; + $('download').disabled = busy || !current || current.stat.directory || current.stream; $('follow').disabled = busy || !current || current.stat.directory; $('rename').disabled = busy || currentPath === '/'; $('delete').disabled = busy || currentPath === '/'; @@ -66,14 +66,14 @@ async function connectedClient() { catch { throw new Error('Unable to reach the file server. Reload to try again.'); } return client; } -async function readPath(path) { +async function readPath(path, metadataOnly = false) { const active = await connectedClient(); - try { return await active.readPath(path); } + try { return await active.readPath(path, metadataOnly); } catch (error) { // A timed-out/closed transport has lost its fids. Retry reads once on a // fresh session. Writes are never replayed after an uncertain result. if (!active.closed) throw error; - return (await connectedClient()).readPath(path); + return (await connectedClient()).readPath(path, metadataOnly); } } function normalizePath(path) { @@ -131,16 +131,16 @@ function breadcrumbs(path) { $('up').href = pathURL(parent); $('up').dataset.path = parent; } -async function navigate(path, historyMode = 'push') { +async function navigate(path, historyMode = 'push', follow = false) { path = normalizePath(path); showView('files'); stopFollow(); status('Loading…'); - const result = await readPath(path); + const result = await readPath(path, follow); current = result; currentPath = path; dirty = false; for (const view of ['files', 'sites', 'services']) $(`view-${view}`).href = viewURL(view); $('path').value = path; - const url = pathURL(path); + const url = pathURL(path) + (follow ? '?follow=1' : ''); if (historyMode === 'push' && location.pathname + location.search + location.hash !== url) history.pushState(null, '', url); document.title = `${path} — 9web`; breadcrumbs(path); @@ -154,6 +154,11 @@ async function navigate(path, historyMode = 'push') { const permissions = document.createElement('td'), name = document.createElement('td'), size = document.createElement('td'); permissions.className = 'mode'; permissions.textContent = mode(entry); name.append(pathLink(entry.name + (entry.directory ? '/' : ''), child(path, entry.name), 'entry' + (entry.directory ? ' directory' : ''))); + if (!entry.directory) { + const followLink = pathLink('follow', child(path, entry.name), 'follow-link'); + followLink.dataset.follow = 'true'; followLink.href += '?follow=1'; + name.append(followLink); + } size.className = 'size'; size.textContent = entry.directory ? '—' : entry.length.toLocaleString(); row.append(permissions, name, size); rows.append(row); } @@ -163,7 +168,7 @@ async function navigate(path, historyMode = 'push') { } else { $('filename').textContent = result.stat.name; $('file-mode').textContent = mode(result.stat); - $('file-size').textContent = `${result.bytes.length.toLocaleString()} bytes`; + $('file-size').textContent = result.stream ? 'stream' : `${result.bytes.length.toLocaleString()} bytes`; let editable = true; try { if (result.bytes.includes(0)) throw new Error('Binary'); @@ -175,7 +180,7 @@ async function navigate(path, historyMode = 'push') { status('File loaded.'); } updateStat(result); - markTree(path); + if (follow && !result.stat.directory) startFollow(); } function mayLeave() { return !dirty || window.confirm('Discard unsaved changes?'); } document.addEventListener('click', event => { @@ -187,19 +192,19 @@ document.addEventListener('click', event => { if (view === activeView) return; history.pushState(null, '', viewURL(view)); showView(view); - if (view === 'files' && !current) action(() => navigate(currentPath, 'none')).then(loadTree); + if (view === 'files' && !current) action(() => navigate(currentPath, 'none')); return; } const link = event.target.closest('a[data-path]'); if (!link || event.button !== 0 || event.ctrlKey || event.metaKey || event.shiftKey || event.altKey) return; event.preventDefault(); - if (!busy && mayLeave()) action(() => navigate(link.dataset.path)); + if (!busy && mayLeave()) action(() => navigate(link.dataset.path, 'push', link.dataset.follow === 'true')); }); $('path-form').onsubmit = event => { event.preventDefault(); if (mayLeave()) action(() => navigate($('path').value)); }; -$('reload').onclick = () => { if (mayLeave()) action(() => navigate(current ? currentPath : locationPath(), 'none')); }; +$('reload').onclick = () => { if (mayLeave()) action(() => navigate(current ? currentPath : locationPath(), 'none', Boolean(current?.stream))); }; $('editor').oninput = () => { dirty = true; controls(); }; $('save').onclick = () => action(async () => { const bytes = new TextEncoder().encode($('editor').value); @@ -224,26 +229,20 @@ window.addEventListener('popstate', () => { const view = locationView(), path = locationPath(); if (busy || (view === 'files' && path !== currentPath && !mayLeave())) { history.pushState(null, '', viewURL(activeView)); return; } showView(view); - if (view === 'files' && (!current || path !== currentPath)) action(() => navigate(path, 'none')).then(loadTree); + const follow = new URLSearchParams(location.search).get('follow') === '1'; + if (view === 'files' && (!current || path !== currentPath || Boolean(current.stream) !== follow)) action(() => navigate(path, 'none', follow)); }); window.addEventListener('beforeunload', event => { if (dirty) { event.preventDefault(); event.returnValue = ''; } }); -// ---- HTTP /fs view: tree, stat panel, create/rename/delete/upload, follow ---- +// ---- HTTP /fs view: stat panel, create/rename/delete/upload, follow ---- // These operate over the gateway's HTTP mapping of the same 9P tree, so they // stay independent of the WebSocket editing session above. function fsURL(path) { return '/fs' + path.split('/').map(encodeURIComponent).join('/'); } function parentOf(path) { return path.replace(/\/[^/]+$/, '') || '/'; } function baseName(path) { return path.split('/').filter(Boolean).pop() || ''; } function dirContext() { return current && current.stat.directory ? currentPath : parentOf(currentPath); } -function sortEntries(a, b) { return Number(b.dir) - Number(a.dir) || a.name.localeCompare(b.name); } - -async function fetchDir(path) { - const response = await fetch(fsURL(path) + (path.endsWith('/') ? '' : '/'), { headers: { accept: 'application/json' } }); - if (!response.ok) throw new Error(`list failed: ${response.status}`); - return response.json(); -} function renderStat(rows) { const dl = $('stat'); @@ -276,72 +275,37 @@ function updateStat(result) { }).catch(() => {}); } -function treeItem(entry, path) { - const li = document.createElement('li'); - const node = document.createElement('span'); - node.className = 'node' + (entry.dir ? ' dir' : ''); - node.dataset.node = path; - const twist = document.createElement('span'); - twist.className = 'twist'; - twist.textContent = entry.dir ? '▸' : ' '; - node.append(twist, document.createTextNode(entry.name)); - li.append(node); - node.onclick = () => { - if (entry.dir) toggleTree(li, path, twist); - else if (!busy && mayLeave()) action(() => navigate(path)); - }; - return li; -} -async function toggleTree(li, path, twist) { - const open = li.querySelector(':scope > ul'); - if (open) { open.remove(); twist.textContent = '▸'; return; } - twist.textContent = '▾'; - const ul = document.createElement('ul'); - li.append(ul); - try { - for (const entry of (await fetchDir(path)).sort(sortEntries)) ul.append(treeItem(entry, child(path, entry.name))); - } catch (error) { ul.textContent = '(unavailable)'; } -} -async function loadTree() { - const root = $('filetree'); - try { - const entries = (await fetchDir('/')).sort(sortEntries); - root.replaceChildren(...entries.map(entry => treeItem(entry, '/' + entry.name))); - } catch { /* upstream not ready; leave the last tree in place */ } -} -function markTree(path) { - for (const node of $('filetree').querySelectorAll('.node.current')) node.classList.remove('current'); - const match = $('filetree').querySelector(`.node[data-node="${CSS.escape(path)}"]`); - if (match) match.classList.add('current'); -} let stream = null; -function stopFollow() { +function stopFollow(clear = true) { if (stream) { stream.close(); stream = null; } $('follow').classList.remove('active'); - $('stream').hidden = true; - $('stream').textContent = ''; + if (clear) { $('stream').hidden = true; $('stream').textContent = ''; } } function startFollow() { if (!current || current.stat.directory) return; + current.stream = true; + history.replaceState(null, '', viewURL('files')); + controls(); $('editor').hidden = true; $('binary').hidden = true; $('edit-actions').hidden = true; const view = $('stream'); view.hidden = false; view.textContent = ''; $('follow').classList.add('active'); + status('Following stream…'); stream = new EventSource(fsURL(currentPath) + '?follow=1'); stream.onmessage = event => { view.textContent += event.data + '\n'; view.scrollTop = view.scrollHeight; }; - stream.addEventListener('eof', stopFollow); - stream.onerror = () => { status('Stream ended.'); stopFollow(); }; + stream.addEventListener('eof', () => { stopFollow(false); status('End of stream.'); }); + stream.onerror = () => { status('Stream ended.'); stopFollow(false); }; } $('follow').onclick = () => { - if ($('follow').classList.contains('active')) { stopFollow(); action(() => navigate(currentPath, 'none')); } + if ($('follow').classList.contains('active')) { stopFollow(false); status('Stream stopped.'); } else startFollow(); }; async function mutate(fn, done) { if (busy) return; busy = true; controls(); - try { await fn(); await loadTree(); status(done); } + try { await fn(); status(done); } catch (error) { status(error.message, true); } finally { busy = false; controls(); } } @@ -406,7 +370,7 @@ async function start() { currentPath = locationPath(); const view = locationView(); showView(view); - if (view === 'files') await action(() => navigate(currentPath, 'none')).then(loadTree); + if (view === 'files') await action(() => navigate(currentPath, 'none', new URLSearchParams(location.search).get('follow') === '1')); } catch (error) { status(error.message, true); } } start(); diff --git a/web/static/catalog.mjs b/web/static/catalog.mjs index d1d3444..e110dee 100644 --- a/web/static/catalog.mjs +++ b/web/static/catalog.mjs @@ -12,6 +12,7 @@ function url(address, port, protocol = 'http') { export function catalogViews(pathLink) { let active = 'files', timer = null, request = null, revision = 0, pending = new Set(); const rows = new Map(); + let namespace = ''; function status(view, text, error = false) { const p = $(`${view}-status`); p.textContent = text; p.classList.toggle('error', error); } @@ -19,7 +20,7 @@ export function catalogViews(pathLink) { if (pending.has(site.name)) return; pending.add(site.name); updateSite(site); try { - const r = await fetch(`/fs/sites/${encodeURIComponent(site.name)}/ctl`, { method: 'PUT', body: command }); + const r = await fetch(`/fs${namespace}/sites/${encodeURIComponent(site.name)}/ctl`, { method: 'PUT', body: command }); if (!r.ok) throw new Error(await r.text()); status('sites', `${site.title}: ${command} requested.`); } catch (error) { status('sites', error.message, true); } @@ -35,7 +36,7 @@ export function catalogViews(pathLink) { const tools = element('div'); tools.className = 'site-actions'; const build = element('button', 'build'), watch = element('button', 'watch'), cancel = element('button', 'cancel'); for (const b of [build, watch, cancel]) b.type = 'button'; - tools.append(build, watch, cancel, pathLink('files', `/sites/${site.name}/public`), pathLink('log', `/sites/${site.name}/log`)); + tools.append(build, watch, cancel, pathLink('files', `${namespace}/sites/${site.name}/public`), pathLink('log', `${namespace}/sites/${site.name}/log`)); actions.append(tools); tr.append(name, phase, finished, actions); row = { tr, link, root, phase, finished, build, watch, cancel }; rows.set(site.name, row); $('sites-entries').append(tr); @@ -74,6 +75,7 @@ export function catalogViews(pathLink) { if (!r.ok) throw new Error(`Unable to load ${view}: HTTP ${r.status}`); const data = await r.json(); if (ticket !== revision || active !== view) return; + namespace = data.namespace || ''; if (view === 'sites') { const names = new Set(data.sites.map(s => s.name)); for (const [name, row] of rows) if (!names.has(name)) { row.tr.remove(); rows.delete(name); } diff --git a/web/static/client.mjs b/web/static/client.mjs index 69f44d2..c012fc0 100644 --- a/web/static/client.mjs +++ b/web/static/client.mjs @@ -96,9 +96,10 @@ export class Client { if (live && !this.closed) await this.ask('clunk', 1); } } - readPath(path) { + readPath(path, metadataOnly = false) { return this.serial(() => this.withFile(path, async () => { const stat = JSON.parse(decoder.decode((await this.ask('stat', 1)).data)); + if (metadataOnly && !stat.directory) return { stat, entries: [], bytes: new Uint8Array(), stream: true }; const opened = await this.ask('open', 1, 0); const count = Math.min(this.wasm.max_read(), opened.number || Infinity); let offset = 0; diff --git a/web/static/index.html b/web/static/index.html index 4ed498d..c392e65 100644 --- a/web/static/index.html +++ b/web/static/index.html @@ -33,10 +33,6 @@ </form> </nav> <div class="layout"> - <aside id="sidebar" aria-label="Tree"> - <div class="tree-head">tree</div> - <ul id="filetree" role="tree"></ul> - </aside> <main> <nav id="breadcrumbs" aria-label="Breadcrumb"><a href="/" data-path="/">root</a></nav> <div class="summary"> diff --git a/web/static/style.css b/web/static/style.css index 53ccda3..e17f1e5 100644 --- a/web/static/style.css +++ b/web/static/style.css @@ -8,6 +8,7 @@ body { margin: 0; padding: 22px 3%; } a { color: #07539b; text-decoration: none; } a:hover { text-decoration: underline; } +.follow-link { margin-left: 14px; font-size: 11px; color: #777; } header { padding: 0 8px 17px; } .brand { font-size: 30px; font-weight: bold; color: #222; letter-spacing: -1px; } .description { color: #777; margin-top: 1px; } @@ -81,17 +82,7 @@ textarea { display: block; width: 100%; min-height: 420px; resize: vertical; bor #binary { color: #777; padding: 16px 8px; } .actions { display: flex; align-items: center; gap: 12px; margin-top: 10px; font-size: 12px; } #edit-status { color: #777; } -.layout { display: flex; align-items: flex-start; gap: 16px; } -#sidebar { flex: 0 0 220px; border-right: 1px solid #e2e2e2; padding: 8px 10px 0 0; min-width: 0; } -.tree-head { color: #999; font-size: 11px; text-transform: uppercase; letter-spacing: .05em; padding: 4px; } -#filetree, #filetree ul { list-style: none; margin: 0; padding: 0; } -#filetree ul { margin-left: 12px; } -#filetree li { line-height: 1.7; white-space: nowrap; } -#filetree .node { cursor: pointer; font-family: monospace; overflow-wrap: anywhere; } -#filetree .node.dir { font-weight: bold; } -#filetree .node.current { background: #ccc; } -#filetree .twist { display: inline-block; width: 12px; color: #999; } -.layout main { flex: 1 1 auto; min-width: 0; } +.layout main { min-width: 0; } #stat-panel { margin-top: 18px; border-top: 1px solid #eee; padding-top: 8px; } #stat-panel h2 { font-size: 12px; text-transform: uppercase; letter-spacing: .05em; color: #999; margin: 0 0 6px; } #stat { display: grid; grid-template-columns: max-content 1fr; gap: 2px 12px; font: 12px monospace; margin: 0; } @@ -122,6 +113,4 @@ a:focus-visible, button:focus-visible { outline: 2px solid #07539b; outline-offs .summary { align-items: start; } .file-heading { gap: 7px 12px; } textarea { min-height: 350px; } - .layout { flex-direction: column; } - #sidebar { flex: none; width: 100%; border-right: 0; border-bottom: 1px solid #e2e2e2; max-height: 180px; overflow: auto; padding: 0 0 6px; } } |
