Add reusable HTTP transport, serial gateway, and WASM file browser

1 files changed, 268 insertions, 0 deletions

diff --git a/test/web/e2e.mjs b/test/web/e2e.mjs
new file mode 100644
index 0000000..2ff83ae
--- /dev/null
+++ b/test/web/e2e.mjs
@@ -0,0 +1,268 @@
+// Real Chromium + native cloud9 HTTPS client + pinned go9p. No npm packages.
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { once } from 'node:events';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import net from 'node:net';
+import tls from 'node:tls';
+import http from 'node:http';
+
+const [bridgeBinary, nativeBinary] = process.argv.slice(2).map(p => path.resolve(p));
+if (!nativeBinary) throw new Error('usage: node test/web/e2e.mjs BRIDGE NATIVE_TEST');
+const root = path.resolve('.');
+await fs.mkdir('.zig-cache/tmp', { recursive: true });
+const work = await fs.mkdtemp(path.join(root, '.zig-cache/e2e-'));
+const children = [], logs = [], sockets = new Set();
+let browserSocket, tlsServer;
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+function start(command, args, options = {}) {
+  const child = spawn(command, args, { ...options, env: { ...process.env, TMPDIR: path.join(root, '.zig-cache/tmp'), ...options.env }, stdio: ['ignore', 'pipe', 'pipe'] });
+  child.output = ''; child.errors = '';
+  child.stdout.on('data', b => { child.output += b; });
+  child.stderr.on('data', b => { child.errors = (child.errors + b).slice(-20000); });
+  child.on('error', error => { child.errors += error.message; });
+  children.push(child); return child;
+}
+async function run(command, args, options = {}, success = true) {
+  const child = start(command, args, options);
+  const timer = setTimeout(() => child.kill('SIGKILL'), 60000);
+  const [code] = await once(child, 'exit'); clearTimeout(timer);
+  if (success) assert.equal(code, 0, `${command}: ${child.errors}`);
+  else assert.notEqual(code, 0, 'untrusted TLS certificate must fail');
+  return child;
+}
+async function wait(check, message, timeout = 15000) {
+  const end = Date.now() + timeout;
+  while (Date.now() < end) { try { const result = await check(); if (result) return result; } catch {} await delay(40); }
+  throw new Error(`Timed out: ${message}`);
+}
+async function launchBridge(upstream, origin, extra = []) {
+  const child = start(bridgeBinary, ['--listen', '127.0.0.1:0', '--upstream', upstream, ...(origin ? ['--origin', origin] : []), ...extra]);
+  // With a public origin the log does not include the bound address. Tests
+  // requiring a proxy instead reserve a port and pass it explicitly below.
+  const match = await wait(() => child.errors.match(/cloud9-http (http:\/\/127\.0\.0\.1:\d+)/), 'bridge ready');
+  return { child, url: match[1] };
+}
+async function port() { const s = net.createServer(); s.listen(0, '127.0.0.1'); await once(s, 'listening'); const p = s.address().port; await new Promise(r => s.close(r)); return p; }
+let nextId = 0; const pending = new Map();
+function cdp(method, params = {}) {
+  const id = ++nextId;
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => { pending.delete(id); reject(new Error(`CDP timeout: ${method}`)); }, 30000);
+    pending.set(id, { resolve, reject, timer }); browserSocket.send(JSON.stringify({ id, method, params }));
+  });
+}
+async function evaluate(expression) {
+  const result = await cdp('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true });
+  if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description || result.exceptionDetails.text);
+  return result.result.value;
+}
+async function click(selector) { await evaluate(`document.querySelector(${JSON.stringify(selector)}).click()`); }
+async function statusIncludes(text) { await wait(() => evaluate(`document.getElementById('status').textContent.includes(${JSON.stringify(text)})`), `status: ${text}`); }
+async function openPath(value) {
+  await evaluate(`document.getElementById('path').value=${JSON.stringify(value)};document.getElementById('path-form').requestSubmit()`);
+}
+try {
+  const fixture = path.join(work, 'fixture');
+  await run('go', ['build', '-o', fixture, './webfixture'], { cwd: path.join(root, 'test/differential') });
+  const server = start(fixture, []);
+  const upstream = await wait(() => server.output.match(/127\.0\.0\.1:\d+/)?.[0], 'go9p ready');
+  const bridge = await launchBridge(`tcp:${upstream}`);
+  const url = bridge.url;
+  const wasmResponse = await fetch(`${url}/_cloud9/cloud9.wasm`);
+  assert.equal(wasmResponse.headers.get('content-type'), 'application/wasm');
+  const module = await WebAssembly.compile(await wasmResponse.arrayBuffer());
+  assert.deepEqual(WebAssembly.Module.imports(module), [], 'WASM uses cloud9 without host imports');
+  assert.equal((await fetch(`${url}/_cloud9/absent`)).status, 404);
+  assert.equal((await fetch(`${url}/`, { method: 'POST' })).status, 405);
+  assert.equal((await fetch(`${url}/_cloud9/9p`)).status, 403);
+  assert.equal(await new Promise((resolve,reject)=>{const r=http.get(url,{headers:{Host:'unrelated.example'}},response=>{response.resume();resolve(response.statusCode)});r.on('error',reject)}), 403);
+  await run(nativeBinary, [`${url}/_cloud9/9p`, url]);
+
+  const debugPort = await port();
+  const chrome = start(process.env.CLOUD9_CHROME || 'google-chrome-stable', ['--headless=new', '--no-sandbox', '--disable-dev-shm-usage', '--no-first-run', '--no-default-browser-check', `--user-data-dir=${work}/chrome`, `--remote-debugging-port=${debugPort}`, 'about:blank']);
+  const target = await wait(async () => {
+    const response = await fetch(`http://127.0.0.1:${debugPort}/json/new?about:blank`, { method: 'PUT' }); return response.ok && response.json();
+  }, 'Chromium ready');
+  browserSocket = new WebSocket(target.webSocketDebuggerUrl);
+  await once(browserSocket, 'open');
+  browserSocket.addEventListener('message', ({ data }) => {
+    const message = JSON.parse(data);
+    if (message.id) {
+      const request = pending.get(message.id); if (!request) return;
+      clearTimeout(request.timer); pending.delete(message.id);
+      if (message.error) request.reject(new Error(JSON.stringify(message.error))); else request.resolve(message.result);
+    } else if (message.method === 'Runtime.exceptionThrown') logs.push(message.params.exceptionDetails);
+  });
+  await cdp('Runtime.enable'); await cdp('Page.enable');
+  await cdp('Browser.setDownloadBehavior', { behavior: 'allow', downloadPath: path.join(work, 'downloads') });
+  await cdp('Emulation.setDeviceMetricsOverride', { width: 1120, height: 900, deviceScaleFactor: 1, mobile: false });
+  await cdp('Page.navigate', { url });
+  await wait(() => evaluate(`typeof document.getElementById('path-form')?.onsubmit === 'function'`), 'UI loaded');
+  await statusIncludes('entries');
+  assert.equal(await evaluate(`document.querySelectorAll('#connect,#disconnect,#user,#tree').length`), 0);
+  assert.deepEqual(await (await fetch(`${url}/_cloud9/config.json`)).json(), {user:'user',tree:''});
+  assert.ok(await evaluate(`[...document.querySelectorAll('.entry')].some(e=>e.textContent==='hello.txt')`));
+  assert.ok(await evaluate(`[...document.querySelectorAll('.entry')].some(e=>e.textContent==='<literal>.txt')`));
+  assert.equal(await evaluate(`document.querySelectorAll('literal').length`), 0);
+  await openPath('/hello.txt'); await statusIncludes('File loaded');
+  assert.equal(await evaluate(`document.getElementById('editor').value`), 'Hello from 9P.\n');
+  assert.equal(await evaluate(`location.pathname`), '/hello.txt');
+  await evaluate('history.back()'); await statusIncludes('entries');
+  await evaluate('history.forward()'); await statusIncludes('File loaded');
+  assert.ok(await evaluate(`document.querySelector('#breadcrumbs a[aria-current]').textContent==='hello.txt'`));
+  await openPath('/notes'); await statusIncludes('151 entries');
+  await openPath('/notes/café.txt'); await statusIncludes('File loaded');
+  assert.equal(await evaluate(`document.getElementById('editor').value`), 'Olá, 世界.\n');
+  await openPath('/edit.txt'); await statusIncludes('File loaded');
+  await evaluate(`document.getElementById('editor').value='Saved from Chromium.\\n';document.getElementById('editor').dispatchEvent(new Event('input'))`);
+  await click('#save'); await statusIncludes('Saved 21 bytes');
+  await evaluate('window.beforeReload=true');
+  await cdp('Page.reload');
+  await wait(() => evaluate(`!window.beforeReload && document.getElementById('status')?.textContent.includes('File loaded')`), 'automatic attach after page reload');
+  assert.equal(await evaluate(`location.pathname`), '/edit.txt');
+  assert.equal(await evaluate(`document.getElementById('editor').value`), 'Saved from Chromium.\n');
+  await openPath('/missing/child'); await statusIncludes('');
+  await wait(() => evaluate(`document.getElementById('status').classList.contains('error') && !document.getElementById('go').disabled`), 'missing path error');
+  await openPath('/hello.txt'); await statusIncludes('File loaded');
+  await openPath('/large.bin'); await statusIncludes('File loaded');
+  assert.equal(await evaluate(`document.getElementById('binary').hidden`), false);
+  assert.equal(await evaluate(`document.getElementById('save').disabled`), true);
+  await click('#download');
+  const download = await wait(async () => {
+    const bytes = await fs.readFile(path.join(work, 'downloads/large.bin'));
+    return bytes.length === 180000 && bytes;
+  }, 'binary download');
+  assert.ok(download.every((value, index) => value === index % 251));
+
+  // Invoke the shipped JS/WASM API in the browser for byte-exact, multi-frame
+  // transfers, queued operations, independent fid spaces, and long walks.
+  const checks = await evaluate(`(async()=>{
+    const {Client}=await import('/_cloud9/client.mjs');
+    const a=await Client.connect(), b=await Client.connect();
+    try {
+      const [large,other]=await Promise.all([a.readPath('/large.bin'),b.readPath('/hello.txt')]);
+      if(large.bytes.length!==180000||large.bytes.some((v,i)=>v!==i%251))throw Error('binary mismatch');
+      const bytes=Uint8Array.from({length:150000},(_,i)=>(i*17)%251);
+      await a.writePath('/edit.txt',bytes);
+      const read=await b.readPath('/edit.txt');
+      if(read.bytes.length!==bytes.length||read.bytes.some((v,i)=>v!==bytes[i]))throw Error('write mismatch');
+      await a.writePath('/edit.txt',new Uint8Array());
+      if((await b.readPath('/edit.txt')).bytes.length!==0)throw Error('truncate mismatch');
+      const deep=await a.readPath('/'+Array(18).fill('deep').join('/')+'/end.txt');
+      if(new TextDecoder().decode(deep.bytes)!=='Deep walk.\\n')throw Error('deep walk');
+      const queued=await Promise.all([a.readPath('/hello.txt'),a.readPath('/notes/café.txt')]);
+      return {binary:large.bytes.length,written:bytes.length,queued:queued.length,independent:new TextDecoder().decode(other.bytes)};
+    }finally{a.close();b.close()}
+  })()`);
+  assert.equal(checks.binary, 180000); assert.equal(checks.written, 150000); assert.equal(checks.queued, 2);
+  await openPath('/'); await statusIncludes('entries');
+  await fs.writeFile(path.join(work, 'browser.png'), Buffer.from((await cdp('Page.captureScreenshot', { format: 'png' })).data, 'base64'));
+  await cdp('Emulation.setDeviceMetricsOverride', { width: 390, height: 844, deviceScaleFactor: 1, mobile: true });
+  assert.ok(await evaluate('document.documentElement.scrollWidth <= innerWidth'), 'mobile layout fits viewport');
+  await fs.writeFile(path.join(work, 'mobile.png'), Buffer.from((await cdp('Page.captureScreenshot', { format: 'png' })).data, 'base64'));
+  assert.deepEqual(logs, [], 'no uncaught browser exceptions');
+
+  // File paths share no routes with gateway assets. Read them through actual
+  // generated links, then reload those URLs in a fresh browser document.
+  for (const name of ['app.mjs', 'style.css', 'client.mjs', 'config.json', 'cloud9.wasm', '9p', 'space #?% ü.txt', 'literal%2F.txt', 'back\\slash.txt']) {
+    await openPath('/'); await statusIncludes('entries');
+    const expectedPath = '/' + encodeURIComponent(name);
+    const href = await evaluate(`document.querySelector('a[data-path='+CSS.escape(${JSON.stringify('/'+name)})+']').getAttribute('href')`);
+    assert.equal(href, expectedPath);
+    await evaluate(`document.querySelector('a[data-path='+CSS.escape(${JSON.stringify('/'+name)})+']').click()`);
+    await statusIncludes('File loaded');
+    assert.equal(await evaluate('location.pathname'), expectedPath);
+    assert.equal(await evaluate('location.search + location.hash'), '');
+    assert.equal(await evaluate(`document.getElementById('editor').value`), 'Path: '+name+'\n');
+    await evaluate('window.beforeReload=true');
+    await cdp('Page.reload');
+    await wait(() => evaluate(`!window.beforeReload && document.getElementById('status')?.textContent.includes('File loaded')`), 'encoded file reload');
+    assert.equal(await evaluate(`document.getElementById('editor').value`), 'Path: '+name+'\n');
+  }
+  await openPath('/_cloud9/app.mjs'); await statusIncludes('File loaded');
+  assert.equal(await evaluate('location.pathname'), '/%5Fcloud9/app.mjs');
+  await evaluate('window.beforeReload=true'); await cdp('Page.reload');
+  await wait(() => evaluate(`!window.beforeReload && document.getElementById('editor')?.value==='Upstream private-looking path.\\n'`), 'escaped gateway-prefix file');
+  for (const path of ['/bad%escape', '/notes%2Fcaf%C3%A9.txt', '/bad%00name']) {
+    await cdp('Page.navigate', {url:url+path});
+    await wait(() => evaluate(`location.pathname===${JSON.stringify(path)} && document.getElementById('status')?.textContent.includes('The URL contains')`), 'invalid path rejected');
+    await click('.brand'); await statusIncludes('entries');
+    assert.equal(await evaluate('location.pathname'), '/');
+  }
+
+  // Expiry is transport bookkeeping: edits survive it, and Save establishes
+  // a fresh session without a user-facing connection flow.
+  const shortBridge = await launchBridge(`tcp:${upstream}`, null, ['--timeout-ms', '1000', '--user', 'user', '--tree', '']);
+  await cdp('Page.navigate', {url:shortBridge.url+'/edit.txt'});
+  await wait(() => evaluate(`location.port===${JSON.stringify(new URL(shortBridge.url).port)} && document.getElementById('status')?.textContent.includes('File loaded')`), 'direct file URL');
+  await evaluate(`document.getElementById('editor').value='After expiry.\\n';document.getElementById('editor').dispatchEvent(new Event('input'))`);
+  await delay(1300);
+  assert.equal(await evaluate(`document.getElementById('editor').value`), 'After expiry.\n');
+  await click('#save'); await statusIncludes('Saved 14 bytes');
+  assert.equal(await evaluate(`document.getElementById('editor').value`), 'After expiry.\n');
+  const configured = await launchBridge(`tcp:${upstream}`, null, ['--user', 'reader', '--tree', 'named-export']);
+  assert.deepEqual(await (await fetch(configured.url+'/_cloud9/config.json')).json(), {user:'reader',tree:'named-export'});
+  await cdp('Page.navigate', {url:'about:blank'});
+
+  // Same public native API over certificate-verified TLS. The TLS terminator
+  // forwards bytes; the 9P server still runs behind the HTTP bridge over TCP.
+  const key = path.join(work, 'key.pem'), cert = path.join(work, 'cert.pem');
+  await run('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-noenc', '-keyout', key, '-out', cert, '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost']);
+  const httpsPort = await port(), backendPort = await port();
+  const origin = `https://localhost:${httpsPort}`;
+  const secureBridge = start(bridgeBinary, ['--listen', `127.0.0.1:${backendPort}`, '--upstream', `tcp:${upstream}`, '--origin', origin]);
+  await wait(() => secureBridge.errors.includes('cloud9-http'), 'TLS backend ready');
+  tlsServer = tls.createServer({ key: await fs.readFile(key), cert: await fs.readFile(cert), ALPNProtocols: ['http/1.1'] }, socket => {
+    const upstreamSocket = net.connect(backendPort, '127.0.0.1');
+    sockets.add(socket); sockets.add(upstreamSocket);
+    socket.on('error', () => upstreamSocket.destroy()); upstreamSocket.on('error', () => socket.destroy());
+    socket.on('close', () => { sockets.delete(socket); upstreamSocket.destroy(); });
+    upstreamSocket.on('close', () => { sockets.delete(upstreamSocket); socket.destroy(); });
+    socket.pipe(upstreamSocket).pipe(socket);
+  });
+  tlsServer.on('tlsClientError', () => {});
+  tlsServer.listen(httpsPort); await once(tlsServer, 'listening');
+  await run(nativeBinary, [`${origin}/_cloud9/9p`, origin, cert]);
+  await run(nativeBinary, [`${origin}/_cloud9/9p`, origin], {}, false);
+  await run(nativeBinary, [`https://127.0.0.1:${httpsPort}/_cloud9/9p`, origin, cert], {}, false);
+
+  const unixPath = path.join(work, '9p.sock');
+  const unixServer = start(fixture, [unixPath]);
+  await wait(() => unixServer.output.includes(unixPath), 'Unix go9p ready');
+  const unixBridge = await launchBridge(`unix:${unixPath}`);
+  await run(nativeBinary, [`${unixBridge.url}/_cloud9/9p`, unixBridge.url]);
+  const serial = start('python3', ['test/web/serial.py', upstream]);
+  const serialPath = await wait(() => serial.output.match(/\/dev\/pts\/\d+/)?.[0], 'serial fixture ready');
+  const serialBridge = await launchBridge(`file:${serialPath}`);
+  await run(nativeBinary, [`${serialBridge.url}/_cloud9/9p`, serialBridge.url]);
+  // Closing a browser connection must cancel a blocked serial read and release
+  // the exclusive device lease before another session attaches.
+  await delay(100);
+  await run(nativeBinary, [`${serialBridge.url}/_cloud9/9p`, serialBridge.url]);
+
+  const deadlineBridge = start(bridgeBinary, ['--listen', '127.0.0.1:0', '--upstream', `tcp:${upstream}`, '--timeout-ms', '100']);
+  const deadlineURL = await wait(() => deadlineBridge.errors.match(/http:\/\/127\.0\.0\.1:\d+/)?.[0], 'deadline bridge');
+  const stalled = net.connect(Number(new URL(deadlineURL).port), '127.0.0.1');
+  await once(stalled, 'connect');
+  stalled.write('GET / HTTP/1.1\r\n');
+  await Promise.race([once(stalled, 'close'), delay(2000).then(()=>{stalled.destroy();throw Error('header deadline did not close connection')})]);
+  const report = { browser: 'Chromium', reference: 'go9p v1.18.0', wasmHostImports: 0, checks, passed: ['clean path links and reloads', 'encoded filename round trips', 'gateway asset name collisions', 'escaped gateway-prefix file', 'invalid path encoding rejected', 'automatic connection', 'bookmarkable file URLs', 'browser back and forward', 'automatic reconnect preserves edits', 'configured attach defaults', 'directory paging', 'UTF-8 paths and contents', 'text save and reconnect', 'binary read/write', 'browser download', 'mobile layout', 'truncate to empty', '18-component walk', 'concurrent sessions', 'queued operations', 'Rerror recovery', 'HTTP routing and origin policy', 'native HTTP', 'native verified HTTPS', 'untrusted certificate rejection', 'hostname mismatch rejection', 'Unix upstream', 'PTY serial upstream', 'serial lease released after disconnect', 'connection deadline', 'overlapping native requests', 'fragmented upstream I/O'] };
+  await fs.writeFile(path.join(work, 'result.json'), JSON.stringify(report, null, 2)+'\n');
+  console.log(`E2E passed: ${report.passed.length} scenarios. Artifacts: ${path.relative(root, work)}`);
+} catch (error) {
+  console.error(error.stack);
+  if (browserSocket?.readyState === WebSocket.OPEN) console.error('Browser state:', await evaluate(`({status:document.getElementById('status')?.textContent,body:document.body?.innerText})`).catch(String));
+  console.error('Browser exceptions:', logs);
+  for (const child of children) if (child.errors) console.error(child.spawnargs.join(' '), '\n', child.errors);
+  process.exitCode = 1;
+} finally {
+  for (const request of pending.values()) clearTimeout(request.timer);
+  browserSocket?.close();
+  for (const socket of sockets) socket.destroy();
+  tlsServer?.close();
+  for (const child of children.reverse()) if (child.exitCode === null) child.kill('SIGTERM');
+  await delay(300);
+  for (const child of children) if (child.exitCode === null) child.kill('SIGKILL');
+}