From f43c1e11b44e2464f0bb0b635e0abcaf3c717e23 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Tue, 28 Jul 2026 23:30:29 -0300 Subject: LSP seam: async execution model, helix keymap, evaluation harness The base every language backend plugs into. Three parts: ASYNC. The core had no request/response shape - every effect was fire-and-forget or instantaneous. A language query is the first thing that answers later, so: Effect .lsp -> shell worker -> Event .lsp_resp. tty.zig uses io.concurrent + the vaxis queue, gui.zig a detached thread + the mutex queue it already had for ptys; web no-ops it. The worker never touches the core (path/source/arg are snapshotted into an LspJob), one query in flight identified by a monotonic id so a second press makes the first answer stale, and no rows is a legal answer. KEYMAP. Helix's, verified against its default.rs rather than recalled. gd/gD/gy/gi/gr and ]d/[d had no conflicts. The SPC letters did, so pardes's own builtins moved instead of helix's: Kill k->q, Del d->wc (closing a pane is a window op, and c is helix's own close), Dump/Restore s?->f?, Tutor ht->T. A three-exception muscle-memory map is not a map. RESULTS ARE +SEARCH ROWS. path:LINE:COL text, absolute. That is what look.zig resolves and n/N step, so one row from a goto jumps and several open a buffer - helix's multi-result picker needed no picker code. Backends supply exactly one function (lsp.query) plus a supports set and a name; the base has none on purpose. zig build lspbench scores them on the same corpus: feature matrix (trusting results, not the supports flag - a claimed-but-empty kind is reported as a false claim), cold and warm latency, peak RSS. Two snapshot scripts moved. leader.snap encoded the old key paths. chordcut.snap's last two steps clicked column 5, which lands on a FILE pane, so 'key c-b' toggled nothing and the typed text was being read as normal-mode keys - the golden recorded no TTY pane and no cat -v output anywhere. Pointing them at an actual shell makes both steps assert what their comments claim, and the tty paste chord is now covered for the first time. --- test/lspbench.zig | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 test/lspbench.zig (limited to 'test/lspbench.zig') diff --git a/test/lspbench.zig b/test/lspbench.zig new file mode 100644 index 00000000..73b82e2b --- /dev/null +++ b/test/lspbench.zig @@ -0,0 +1,264 @@ +//! The language-backend scoreboard: one binary, three numbers. +//! +//! zig build lspbench -- latency + feature matrix over pardes's own src/ +//! zig build lspbench -- --json -- the same, machine-readable +//! +//! Every competing backend links the same harness against the same corpus and +//! the same query list, so the columns mean the same thing across all of them. +//! What it measures, in the order the evaluation weighs it: +//! +//! FEATURES which lsp.Kind values actually return rows. Claiming support in +//! `lsp.supports` and returning nothing is a FALSE claim and shows +//! up as `claimed-empty` — the harness trusts results, not flags. +//! LATENCY cold (first query, index build included) and warm (median of N) +//! per kind. Cold is what a keypress costs the first time; warm is +//! what it costs forever after. Both matter and they differ by +//! orders of magnitude for an indexing backend. +//! MEMORY peak RSS delta across the run, read from /proc. +//! +//! Lines of code is not measured here — it is `jj diff --stat` against the +//! base commit, which is the honest number (a backend that vendors a library +//! pays for what it vendors only in build time, not in code we maintain). +const std = @import("std"); +const libc = std.c; +const lsp = @import("pardes").lsp; + +// 0.16 slimmed std.fs (no cwd(), no realpathAlloc) and this repo goes through +// libc everywhere for exactly that reason — see look.zig. +extern "c" fn realpath(path: [*:0]const u8, resolved: [*]u8) ?[*:0]u8; + +/// std.time.Timer is gone in 0.16; clock_gettime is what dump.zig already uses. +fn nowNs() u64 { + var ts: std.c.timespec = undefined; + _ = std.c.clock_gettime(.MONOTONIC, &ts); + return @as(u64, @intCast(ts.sec)) *| 1_000_000_000 +| @as(u64, @intCast(ts.nsec)); +} + +pub const std_options: std.Options = .{ .log_level = .err }; + +/// One probe. The corpus is pardes's own source: real Zig, in this repo, that +/// every implementation can reach without a fixture tree to keep in sync. +/// +/// The cursor is pinned by SYMBOL, not by line — the needle is searched for at +/// startup — so editing pardes.zig cannot silently rot the bench into probing +/// a blank line and calling the result "no support". +/// +/// `expect` is a substring the rows must contain for the probe to count as +/// CORRECT rather than merely non-empty. A backend that returns a plausible +/// wrong location scores worse than one that returns nothing, and only this +/// field can tell those two apart. +const Anchor = struct { + file: []const u8, + /// the exact source text to put the cursor on; the FIRST occurrence wins + needle: []const u8, + /// byte offset into `needle` where the cursor sits + at: u32 = 0, + kind: lsp.Kind, + expect: []const u8 = "", +}; + +const anchors = [_]Anchor{ + // a call to a function defined in the same file + .{ .file = "src/pardes.zig", .needle = "modal.hxLineCount(text)", .at = 6, .kind = .definition, .expect = "modal.zig" }, + // a plain local/decl reference within one file + .{ .file = "src/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .definition, .expect = "lsp.zig" }, + // a std reference: needs the zig lib dir, which is the first thing a + // single-file backend cannot do + .{ .file = "src/lsp.zig", .needle = "std.mem.count(u8", .at = 8, .kind = .definition, .expect = "mem.zig" }, + // hover over the same symbol + .{ .file = "src/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .hover, .expect = "" }, + // the file's own symbols + .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .document_symbols, .expect = "Kind" }, + .{ .file = "src/modal.zig", .needle = "pub fn ", .at = 7, .kind = .document_symbols, .expect = "" }, + // references to a symbol used in several places + .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .references, .expect = "" }, + // diagnostics on a file that should have none + .{ .file = "src/lsp.zig", .needle = "const std", .at = 6, .kind = .diagnostics, .expect = "" }, + // the remaining kinds, probed once each so the matrix is complete + .{ .file = "src/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .declaration, .expect = "" }, + .{ .file = "src/lsp.zig", .needle = "out: *std.ArrayList(u8)", .at = 10, .kind = .type_definition, .expect = "" }, + .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .implementation, .expect = "" }, + .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .select_refs, .expect = "" }, + .{ .file = "src/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .format, .expect = "" }, + .{ .file = "src/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .code_action, .expect = "" }, + .{ .file = "src/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .rename, .expect = "" }, + .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .workspace_symbols, .expect = "" }, + .{ .file = "src/lsp.zig", .needle = "const std", .at = 6, .kind = .workspace_diagnostics, .expect = "" }, +}; + +const Result = struct { + kind: lsp.Kind, + /// no rows came back + empty: bool = true, + /// rows came back AND contained `expect` (or nothing was expected) + correct: bool = false, + rows: usize = 0, + cold_us: u64 = 0, + warm_us: u64 = 0, +}; + +const warm_iters = 20; + +pub fn main(init: std.process.Init) !void { + // page_allocator like the other harnesses: this measures the BACKEND, and + // a debug allocator's bookkeeping would land in every number. + const gpa = std.heap.page_allocator; + + const args = try init.minimal.args.toSlice(init.arena.allocator()); + var json = false; + var root: []const u8 = "."; + for (args[1..]) |a| { + if (std.mem.eql(u8, a, "--json")) json = true else root = a; + } + + const rss0 = rssKib(); + var results: std.ArrayList(Result) = .empty; + defer results.deinit(gpa); + + for (anchors) |an| { + const path = try std.fs.path.join(gpa, &.{ root, an.file }); + defer gpa.free(path); + const src = readZ(gpa, path) catch continue; + defer gpa.free(src); + const off = std.mem.indexOf(u8, src, an.needle) orelse { + std.debug.print("bench: anchor not found in {s}: `{s}`\n", .{ an.file, an.needle }); + continue; + }; + var realbuf: [4096]u8 = undefined; + var pz: [4096:0]u8 = undefined; + const pz_s = std.fmt.bufPrintSentinel(&pz, "{s}", .{path}, 0) catch continue; + const abs = std.mem.span(realpath(pz_s.ptr, &realbuf) orelse continue); + const dir = std.fs.path.dirname(abs) orelse "/"; + const req: lsp.Req = .{ + .kind = an.kind, + .path = abs, + .source = src, + .offset = @intCast(off + an.at), + .arg = if (an.kind == .rename) "renamed_by_bench" else if (an.kind == .workspace_symbols) "Kind" else "", + .root = dir, + }; + + var r: Result = .{ .kind = an.kind }; + // COLD: whatever the backend has to build the first time counts. + r.cold_us = runOnce(gpa, req, &r); + // WARM: median of warm_iters, so one scheduling hiccup cannot flatter + // or damn a backend. + var samples: [warm_iters]u64 = undefined; + for (&samples) |*s| { + var throwaway: Result = .{ .kind = an.kind }; + s.* = runOnce(gpa, req, &throwaway); + } + std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); + r.warm_us = samples[warm_iters / 2]; + r.correct = !r.empty and (an.expect.len == 0 or blk: { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + var arena: std.heap.ArenaAllocator = .init(gpa); + defer arena.deinit(); + lsp.query(gpa, arena.allocator(), req, &out); + break :blk std.mem.indexOf(u8, out.items, an.expect) != null; + }); + try results.append(gpa, r); + } + + const rss = rssKib() -| rss0; + if (json) reportJson(results.items, rss) else reportText(results.items, rss); +} + +fn runOnce(gpa: std.mem.Allocator, req: lsp.Req, r: *Result) u64 { + var arena: std.heap.ArenaAllocator = .init(gpa); + defer arena.deinit(); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + const t0 = nowNs(); + lsp.query(gpa, arena.allocator(), req, &out); + const ns = nowNs() -| t0; + if (out.items.len > 0) { + r.empty = false; + r.rows = std.mem.count(u8, out.items, "\n"); + } + return ns / 1000; +} + +fn reportText(rs: []const Result, rss: u64) void { + const o = std.debug.print; + o("backend: {s}\n\n", .{lsp.backend_name}); + o("{s:<24} {s:>9} {s:>7} {s:>10} {s:>10} {s}\n", .{ "kind", "supports", "rows", "cold(us)", "warm(us)", "verdict" }); + o("{s}\n", .{"-" ** 78}); + var claimed: usize = 0; + var working: usize = 0; + var lies: usize = 0; + for (rs) |r| { + const sup = lsp.supports.contains(r.kind); + if (sup) claimed += 1; + const verdict: []const u8 = if (!sup and r.empty) + "-" + else if (sup and r.empty) + "CLAIMED-EMPTY" + else if (!sup and !r.empty) + "unclaimed-works" + else if (r.correct) + "ok" + else + "rows-but-wrong"; + if (sup and !r.empty and r.correct) working += 1; + if (sup and r.empty) lies += 1; + o("{s:<24} {s:>9} {d:>7} {d:>10} {d:>10} {s}\n", .{ + @tagName(r.kind), if (sup) "yes" else "no", r.rows, r.cold_us, r.warm_us, verdict, + }); + } + o("\nclaimed {d}, verified working {d}, false claims {d}\n", .{ claimed, working, lies }); + o("peak rss delta: {d} KiB\n", .{rss}); +} + +fn reportJson(rs: []const Result, rss: u64) void { + const o = std.debug.print; + o("{{\"backend\":\"{s}\",\"rss_kib\":{d},\"probes\":[", .{ lsp.backend_name, rss }); + for (rs, 0..) |r, i| { + if (i > 0) o(",", .{}); + o("{{\"kind\":\"{s}\",\"supports\":{},\"rows\":{d},\"correct\":{},\"cold_us\":{d},\"warm_us\":{d}}}", .{ + @tagName(r.kind), lsp.supports.contains(r.kind), r.rows, r.correct, r.cold_us, r.warm_us, + }); + } + o("]}}\n", .{}); +} + +fn readZ(gpa: std.mem.Allocator, path: []const u8) ![:0]u8 { + var pathbuf: [4096]u8 = undefined; + const path_z = try std.fmt.bufPrintSentinel(&pathbuf, "{s}", .{path}, 0); + const fd = libc.open(path_z, .{ .ACCMODE = .RDONLY }); + if (fd < 0) return error.OpenFailed; + defer _ = libc.close(fd); + var buf: std.ArrayList(u8) = .empty; + errdefer buf.deinit(gpa); + var chunk: [16384]u8 = undefined; + while (true) { + const n = libc.read(fd, &chunk, chunk.len); + if (n < 0) { + if (libc.errno(n) == .INTR) continue; + return error.ReadFailed; + } + if (n == 0) break; + try buf.appendSlice(gpa, chunk[0..@intCast(n)]); + } + return buf.toOwnedSliceSentinel(gpa, 0); +} + +/// VmHWM from /proc/self/status — the peak, not the current, so a backend that +/// frees its index before returning still pays for having built it. +fn rssKib() u64 { + const fd = libc.open("/proc/self/status", .{ .ACCMODE = .RDONLY }); + if (fd < 0) return 0; + defer _ = libc.close(fd); + var buf: [4096]u8 = undefined; + const n = libc.read(fd, &buf, buf.len); + if (n <= 0) return 0; + var it = std.mem.tokenizeScalar(u8, buf[0..@intCast(n)], '\n'); + while (it.next()) |line| { + if (!std.mem.startsWith(u8, line, "VmHWM:")) continue; + var t = std.mem.tokenizeAny(u8, line["VmHWM:".len..], " \t"); + const v = t.next() orelse return 0; + return std.fmt.parseInt(u64, v, 10) catch 0; + } + return 0; +} -- cgit v1.3