summaryrefslogtreecommitdiff
path: root/test/lspbench.zig
diff options
context:
space:
mode:
Diffstat (limited to 'test/lspbench.zig')
-rw-r--r--test/lspbench.zig264
1 files changed, 264 insertions, 0 deletions
diff --git a/test/lspbench.zig b/test/lspbench.zig
new file mode 100644
index 00000000..73b82e2b
--- /dev/null
+++ b/test/lspbench.zig
@@ -0,0 +1,264 @@
+//! The language-backend scoreboard: one binary, three numbers.
+//!
+//! zig build lspbench -- latency + feature matrix over pardes's own src/
+//! zig build lspbench -- --json -- the same, machine-readable
+//!
+//! Every competing backend links the same harness against the same corpus and
+//! the same query list, so the columns mean the same thing across all of them.
+//! What it measures, in the order the evaluation weighs it:
+//!
+//! FEATURES which lsp.Kind values actually return rows. Claiming support in
+//! `lsp.supports` and returning nothing is a FALSE claim and shows
+//! up as `claimed-empty` — the harness trusts results, not flags.
+//! LATENCY cold (first query, index build included) and warm (median of N)
+//! per kind. Cold is what a keypress costs the first time; warm is
+//! what it costs forever after. Both matter and they differ by
+//! orders of magnitude for an indexing backend.
+//! MEMORY peak RSS delta across the run, read from /proc.
+//!
+//! Lines of code is not measured here — it is `jj diff --stat` against the
+//! base commit, which is the honest number (a backend that vendors a library
+//! pays for what it vendors only in build time, not in code we maintain).
+const std = @import("std");
+const libc = std.c;
+const lsp = @import("pardes").lsp;
+
+// 0.16 slimmed std.fs (no cwd(), no realpathAlloc) and this repo goes through
+// libc everywhere for exactly that reason — see look.zig.
+extern "c" fn realpath(path: [*:0]const u8, resolved: [*]u8) ?[*:0]u8;
+
+/// std.time.Timer is gone in 0.16; clock_gettime is what dump.zig already uses.
+fn nowNs() u64 {
+ var ts: std.c.timespec = undefined;
+ _ = std.c.clock_gettime(.MONOTONIC, &ts);
+ return @as(u64, @intCast(ts.sec)) *| 1_000_000_000 +| @as(u64, @intCast(ts.nsec));
+}
+
+pub const std_options: std.Options = .{ .log_level = .err };
+
+/// One probe. The corpus is pardes's own source: real Zig, in this repo, that
+/// every implementation can reach without a fixture tree to keep in sync.
+///
+/// The cursor is pinned by SYMBOL, not by line — the needle is searched for at
+/// startup — so editing pardes.zig cannot silently rot the bench into probing
+/// a blank line and calling the result "no support".
+///
+/// `expect` is a substring the rows must contain for the probe to count as
+/// CORRECT rather than merely non-empty. A backend that returns a plausible
+/// wrong location scores worse than one that returns nothing, and only this
+/// field can tell those two apart.
+const Anchor = struct {
+ file: []const u8,
+ /// the exact source text to put the cursor on; the FIRST occurrence wins
+ needle: []const u8,
+ /// byte offset into `needle` where the cursor sits
+ at: u32 = 0,
+ kind: lsp.Kind,
+ expect: []const u8 = "",
+};
+
+const anchors = [_]Anchor{
+ // a call to a function defined in the same file
+ .{ .file = "src/pardes.zig", .needle = "modal.hxLineCount(text)", .at = 6, .kind = .definition, .expect = "modal.zig" },
+ // a plain local/decl reference within one file
+ .{ .file = "src/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .definition, .expect = "lsp.zig" },
+ // a std reference: needs the zig lib dir, which is the first thing a
+ // single-file backend cannot do
+ .{ .file = "src/lsp.zig", .needle = "std.mem.count(u8", .at = 8, .kind = .definition, .expect = "mem.zig" },
+ // hover over the same symbol
+ .{ .file = "src/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .hover, .expect = "" },
+ // the file's own symbols
+ .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .document_symbols, .expect = "Kind" },
+ .{ .file = "src/modal.zig", .needle = "pub fn ", .at = 7, .kind = .document_symbols, .expect = "" },
+ // references to a symbol used in several places
+ .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .references, .expect = "" },
+ // diagnostics on a file that should have none
+ .{ .file = "src/lsp.zig", .needle = "const std", .at = 6, .kind = .diagnostics, .expect = "" },
+ // the remaining kinds, probed once each so the matrix is complete
+ .{ .file = "src/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .declaration, .expect = "" },
+ .{ .file = "src/lsp.zig", .needle = "out: *std.ArrayList(u8)", .at = 10, .kind = .type_definition, .expect = "" },
+ .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .implementation, .expect = "" },
+ .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .select_refs, .expect = "" },
+ .{ .file = "src/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .format, .expect = "" },
+ .{ .file = "src/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .code_action, .expect = "" },
+ .{ .file = "src/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .rename, .expect = "" },
+ .{ .file = "src/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .workspace_symbols, .expect = "" },
+ .{ .file = "src/lsp.zig", .needle = "const std", .at = 6, .kind = .workspace_diagnostics, .expect = "" },
+};
+
+const Result = struct {
+ kind: lsp.Kind,
+ /// no rows came back
+ empty: bool = true,
+ /// rows came back AND contained `expect` (or nothing was expected)
+ correct: bool = false,
+ rows: usize = 0,
+ cold_us: u64 = 0,
+ warm_us: u64 = 0,
+};
+
+const warm_iters = 20;
+
+pub fn main(init: std.process.Init) !void {
+ // page_allocator like the other harnesses: this measures the BACKEND, and
+ // a debug allocator's bookkeeping would land in every number.
+ const gpa = std.heap.page_allocator;
+
+ const args = try init.minimal.args.toSlice(init.arena.allocator());
+ var json = false;
+ var root: []const u8 = ".";
+ for (args[1..]) |a| {
+ if (std.mem.eql(u8, a, "--json")) json = true else root = a;
+ }
+
+ const rss0 = rssKib();
+ var results: std.ArrayList(Result) = .empty;
+ defer results.deinit(gpa);
+
+ for (anchors) |an| {
+ const path = try std.fs.path.join(gpa, &.{ root, an.file });
+ defer gpa.free(path);
+ const src = readZ(gpa, path) catch continue;
+ defer gpa.free(src);
+ const off = std.mem.indexOf(u8, src, an.needle) orelse {
+ std.debug.print("bench: anchor not found in {s}: `{s}`\n", .{ an.file, an.needle });
+ continue;
+ };
+ var realbuf: [4096]u8 = undefined;
+ var pz: [4096:0]u8 = undefined;
+ const pz_s = std.fmt.bufPrintSentinel(&pz, "{s}", .{path}, 0) catch continue;
+ const abs = std.mem.span(realpath(pz_s.ptr, &realbuf) orelse continue);
+ const dir = std.fs.path.dirname(abs) orelse "/";
+ const req: lsp.Req = .{
+ .kind = an.kind,
+ .path = abs,
+ .source = src,
+ .offset = @intCast(off + an.at),
+ .arg = if (an.kind == .rename) "renamed_by_bench" else if (an.kind == .workspace_symbols) "Kind" else "",
+ .root = dir,
+ };
+
+ var r: Result = .{ .kind = an.kind };
+ // COLD: whatever the backend has to build the first time counts.
+ r.cold_us = runOnce(gpa, req, &r);
+ // WARM: median of warm_iters, so one scheduling hiccup cannot flatter
+ // or damn a backend.
+ var samples: [warm_iters]u64 = undefined;
+ for (&samples) |*s| {
+ var throwaway: Result = .{ .kind = an.kind };
+ s.* = runOnce(gpa, req, &throwaway);
+ }
+ std.mem.sort(u64, &samples, {}, std.sort.asc(u64));
+ r.warm_us = samples[warm_iters / 2];
+ r.correct = !r.empty and (an.expect.len == 0 or blk: {
+ var out: std.ArrayList(u8) = .empty;
+ defer out.deinit(gpa);
+ var arena: std.heap.ArenaAllocator = .init(gpa);
+ defer arena.deinit();
+ lsp.query(gpa, arena.allocator(), req, &out);
+ break :blk std.mem.indexOf(u8, out.items, an.expect) != null;
+ });
+ try results.append(gpa, r);
+ }
+
+ const rss = rssKib() -| rss0;
+ if (json) reportJson(results.items, rss) else reportText(results.items, rss);
+}
+
+fn runOnce(gpa: std.mem.Allocator, req: lsp.Req, r: *Result) u64 {
+ var arena: std.heap.ArenaAllocator = .init(gpa);
+ defer arena.deinit();
+ var out: std.ArrayList(u8) = .empty;
+ defer out.deinit(gpa);
+ const t0 = nowNs();
+ lsp.query(gpa, arena.allocator(), req, &out);
+ const ns = nowNs() -| t0;
+ if (out.items.len > 0) {
+ r.empty = false;
+ r.rows = std.mem.count(u8, out.items, "\n");
+ }
+ return ns / 1000;
+}
+
+fn reportText(rs: []const Result, rss: u64) void {
+ const o = std.debug.print;
+ o("backend: {s}\n\n", .{lsp.backend_name});
+ o("{s:<24} {s:>9} {s:>7} {s:>10} {s:>10} {s}\n", .{ "kind", "supports", "rows", "cold(us)", "warm(us)", "verdict" });
+ o("{s}\n", .{"-" ** 78});
+ var claimed: usize = 0;
+ var working: usize = 0;
+ var lies: usize = 0;
+ for (rs) |r| {
+ const sup = lsp.supports.contains(r.kind);
+ if (sup) claimed += 1;
+ const verdict: []const u8 = if (!sup and r.empty)
+ "-"
+ else if (sup and r.empty)
+ "CLAIMED-EMPTY"
+ else if (!sup and !r.empty)
+ "unclaimed-works"
+ else if (r.correct)
+ "ok"
+ else
+ "rows-but-wrong";
+ if (sup and !r.empty and r.correct) working += 1;
+ if (sup and r.empty) lies += 1;
+ o("{s:<24} {s:>9} {d:>7} {d:>10} {d:>10} {s}\n", .{
+ @tagName(r.kind), if (sup) "yes" else "no", r.rows, r.cold_us, r.warm_us, verdict,
+ });
+ }
+ o("\nclaimed {d}, verified working {d}, false claims {d}\n", .{ claimed, working, lies });
+ o("peak rss delta: {d} KiB\n", .{rss});
+}
+
+fn reportJson(rs: []const Result, rss: u64) void {
+ const o = std.debug.print;
+ o("{{\"backend\":\"{s}\",\"rss_kib\":{d},\"probes\":[", .{ lsp.backend_name, rss });
+ for (rs, 0..) |r, i| {
+ if (i > 0) o(",", .{});
+ o("{{\"kind\":\"{s}\",\"supports\":{},\"rows\":{d},\"correct\":{},\"cold_us\":{d},\"warm_us\":{d}}}", .{
+ @tagName(r.kind), lsp.supports.contains(r.kind), r.rows, r.correct, r.cold_us, r.warm_us,
+ });
+ }
+ o("]}}\n", .{});
+}
+
+fn readZ(gpa: std.mem.Allocator, path: []const u8) ![:0]u8 {
+ var pathbuf: [4096]u8 = undefined;
+ const path_z = try std.fmt.bufPrintSentinel(&pathbuf, "{s}", .{path}, 0);
+ const fd = libc.open(path_z, .{ .ACCMODE = .RDONLY });
+ if (fd < 0) return error.OpenFailed;
+ defer _ = libc.close(fd);
+ var buf: std.ArrayList(u8) = .empty;
+ errdefer buf.deinit(gpa);
+ var chunk: [16384]u8 = undefined;
+ while (true) {
+ const n = libc.read(fd, &chunk, chunk.len);
+ if (n < 0) {
+ if (libc.errno(n) == .INTR) continue;
+ return error.ReadFailed;
+ }
+ if (n == 0) break;
+ try buf.appendSlice(gpa, chunk[0..@intCast(n)]);
+ }
+ return buf.toOwnedSliceSentinel(gpa, 0);
+}
+
+/// VmHWM from /proc/self/status — the peak, not the current, so a backend that
+/// frees its index before returning still pays for having built it.
+fn rssKib() u64 {
+ const fd = libc.open("/proc/self/status", .{ .ACCMODE = .RDONLY });
+ if (fd < 0) return 0;
+ defer _ = libc.close(fd);
+ var buf: [4096]u8 = undefined;
+ const n = libc.read(fd, &buf, buf.len);
+ if (n <= 0) return 0;
+ var it = std.mem.tokenizeScalar(u8, buf[0..@intCast(n)], '\n');
+ while (it.next()) |line| {
+ if (!std.mem.startsWith(u8, line, "VmHWM:")) continue;
+ var t = std.mem.tokenizeAny(u8, line["VmHWM:".len..], " \t");
+ const v = t.next() orelse return 0;
+ return std.fmt.parseInt(u64, v, 10) catch 0;
+ }
+ return 0;
+}
generated by cgit v1.3 (git 2.53.0) at 2026-10-03 07:08:02 +0000