summaryrefslogtreecommitdiff
path: root/test/lspbench.zig
diff options
context:
space:
mode:
Diffstat (limited to 'test/lspbench.zig')
-rw-r--r--test/lspbench.zig266
1 files changed, 130 insertions, 136 deletions
diff --git a/test/lspbench.zig b/test/lspbench.zig
index 964d3c8c..59cac155 100644
--- a/test/lspbench.zig
+++ b/test/lspbench.zig
@@ -1,33 +1,9 @@
-//! The language-backend scoreboard: one binary, three numbers.
-//!
-//! zig build lspbench -- latency + feature matrix over pardes's own src/
-//! zig build lspbench -- --json -- the same, machine-readable
-//!
-//! Every competing backend links the same harness against the same corpus and
-//! the same query list, so the columns mean the same thing across all of them.
-//! What it measures, in the order the evaluation weighs it:
-//!
-//! FEATURES which lsp.Kind values actually return rows. Claiming support in
-//! `lsp.supports` and returning nothing is a FALSE claim and shows
-//! up as `claimed-empty` — the harness trusts results, not flags.
-//! LATENCY cold (first query, index build included) and warm (median of N)
-//! per kind. Cold is what a keypress costs the first time; warm is
-//! what it costs forever after. Both matter and they differ by
-//! orders of magnitude for an indexing backend.
-//! MEMORY peak RSS delta across the run, read from /proc.
-//!
-//! Lines of code is not measured here — it is `jj diff --stat` against the
-//! base commit, which is the honest number (a backend that vendors a library
-//! pays for what it vendors only in build time, not in code we maintain).
const std = @import("std");
const libc = std.c;
const lsp = @import("pardes").lsp;
-// 0.16 slimmed std.fs (no cwd(), no realpathAlloc) and this repo goes through
-// libc everywhere for exactly that reason — see look.zig.
extern "c" fn realpath(path: [*:0]const u8, resolved: [*]u8) ?[*:0]u8;
-/// std.time.Timer is gone in 0.16; clock_gettime is what dump.zig already uses.
fn nowNs() u64 {
var ts: std.c.timespec = undefined;
_ = std.c.clock_gettime(.MONOTONIC, &ts);
@@ -36,53 +12,25 @@ fn nowNs() u64 {
pub const std_options: std.Options = .{ .log_level = .err };
-/// One probe. The corpus is pardes's own source: real Zig, in this repo, that
-/// every implementation can reach without a fixture tree to keep in sync.
-///
-/// The cursor is pinned by SYMBOL, not by line — the needle is searched for at
-/// startup — so editing pardes.zig cannot silently rot the bench into probing
-/// a blank line and calling the result "no support".
-///
-/// `expect` is a substring the rows must contain for the probe to count as
-/// CORRECT rather than merely non-empty. A backend that returns a plausible
-/// wrong location scores worse than one that returns nothing, and only this
-/// field can tell those two apart.
+// Coordinates are byte offsets within the first needle occurrence, not source line numbers.
const Anchor = struct {
file: []const u8,
- /// the exact source text to put the cursor on; the FIRST occurrence wins
needle: []const u8,
- /// byte offset into `needle` where the cursor sits
at: u32 = 0,
kind: lsp.Kind,
expect: []const u8 = "",
};
const anchors = [_]Anchor{
- // a call to a function defined in the same file
- .{ .file = "src/pardes.zig", .needle = "modal.hxLineCount(text)", .at = 6, .kind = .definition, .expect = "modal.zig" },
- // a plain local/decl reference within one file
+ .{ .file = "src/pardes.zig", .needle = "modal.cursorLineCount(text)", .at = 6, .kind = .definition, .expect = "modal.zig" },
.{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .definition, .expect = "lsp.zig" },
- // a std reference: needs the zig lib dir, which is the first thing a
- // single-file backend cannot do
.{ .file = "src/lsp/lsp.zig", .needle = "std.mem.count(u8", .at = 8, .kind = .definition, .expect = "mem.zig" },
- // hover over the same symbol
.{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .hover, .expect = "" },
- // the file's own symbols
.{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .document_symbols, .expect = "Kind" },
.{ .file = "src/modal.zig", .needle = "pub fn ", .at = 7, .kind = .document_symbols, .expect = "" },
- // references to a symbol used in several places
.{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .references, .expect = "" },
- // Diagnostics and format probe a DELIBERATELY BROKEN fixture, never the
- // real source. On a clean corpus the correct answer is nothing, which is
- // byte-identical to "this backend has no diagnostics" — a blind spot that
- // rewards emitting a filler row and punishes honesty. test/lspfixture has
- // an unused local (semantic: needs a compiler front end, not a tokenizer)
- // and a misformatted fn (needs a formatter), so both probes have real work.
.{ .file = "test/lspfixture/broken.zig", .needle = "unused_local", .at = 0, .kind = .diagnostics, .expect = "broken.zig" },
- // the remaining kinds, probed once each so the matrix is complete
.{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .declaration, .expect = "" },
- // cursor on `out`, whose type is `*std.Io.Writer` — a type_definition probe
- // wants a NAME whose type must be resolved, not the type spelled out
.{ .file = "src/lsp/lsp.zig", .needle = "out: *std.Io.Writer", .at = 0, .kind = .type_definition, .expect = "" },
.{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .implementation, .expect = "" },
.{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .select_refs, .expect = "" },
@@ -91,131 +39,179 @@ const anchors = [_]Anchor{
.{ .file = "src/lsp/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .rename, .expect = "" },
.{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .workspace_symbols, .expect = "" },
.{ .file = "test/lspfixture/broken.zig", .needle = "unused_local", .at = 0, .kind = .workspace_diagnostics, .expect = "broken.zig" },
- // The three shapes a `.` can have, each with the cursor IMMEDIATELY AFTER
- // the dot, which is where Tab asks from.
- // a field access into std: `std.` lists what `std` declares, the same
- // "can this backend reach the stdlib at all" test `gd` gets.
.{ .file = "src/lsp/lsp.zig", .needle = "std.mem.count(u8", .at = 4, .kind = .completion, .expect = "std.zig" },
- // an enum literal in a FINISHED switch arm. The tree parses, so this
- // probes the expected-type resolution and nothing else.
.{ .file = "src/lsp/lsp_zls.zig", .needle = ".declaration => try rowForToken", .at = 1, .kind = .completion, .expect = "type_definition" },
- // the same thing on the BIGGEST file in the tree. This one is here for
- // its latency column rather than its rows: a completion parses the
- // buffer once per placeholder spelling it tries, so pardes.zig is where
- // that shows up and where the figure quoted in docs/lsp.md comes from.
.{ .file = "src/pardes.zig", .needle = ".definition => .definition,", .at = 1, .kind = .completion, .expect = "" },
- // an enum literal in a HALF-TYPED switch arm, which is what a real
- // keypress looks like: the file does not parse and the switch is not in
- // the tree at all. `verdigris` appears only on the enum member's own
- // line, so a row carrying it came from the DEFINITION and not from the
- // line the cursor sits on — and it has to appear TWICE, because a
- // completion row carries the candidate's own name (the word that would go
- // after the dot) before the declaration line it was read off.
+ // The repeated name proves the result includes the declaration, not just the cursor line.
.{ .file = "test/lspfixture/dotcomplete.zig", .needle = "return switch (s) {\n .", .at = 29, .kind = .completion, .expect = "verdigris verdigris," },
- // ...and the same dot on a line that is ALSO missing its terminator,
- // which is one parse error repaired or not. This is the probe that
- // catches a repair chosen by error count rather than by whether the dot
- // became reachable in the tree.
- // (the needle carries its indent: dothalf.zig's own header explains the
- // shape and would otherwise be the first match)
+ // Indentation keeps the fixture header from becoming the first needle match.
.{ .file = "test/lspfixture/dothalf.zig", .needle = " const z: Shade = .", .at = 22, .kind = .completion, .expect = "verdigris" },
};
const Result = struct {
kind: lsp.Kind,
- /// no rows came back
empty: bool = true,
- /// rows came back AND contained `expect` (or nothing was expected)
correct: bool = false,
rows: usize = 0,
cold_us: u64 = 0,
warm_us: u64 = 0,
+
+ fn accept(r: *Result, output: []const u8, expected: []const u8) void {
+ r.empty = output.len == 0;
+ r.rows = std.mem.count(u8, output, "\n");
+ r.correct = !r.empty and (expected.len == 0 or std.mem.indexOf(u8, output, expected) != null);
+ }
};
const warm_iters = 20;
+const Options = struct {
+ json: bool = false,
+ check: bool = false,
+ root: []const u8 = ".",
+
+ fn parse(args: []const []const u8) !Options {
+ var options: Options = .{};
+ var has_root = false;
+ for (args) |arg| {
+ if (std.mem.eql(u8, arg, "--json")) {
+ options.json = true;
+ } else if (std.mem.eql(u8, arg, "--check")) {
+ options.check = true;
+ } else if (std.mem.startsWith(u8, arg, "-")) {
+ return error.UnknownArgument;
+ } else {
+ if (has_root) return error.MultipleRoots;
+ options.root = arg;
+ has_root = true;
+ }
+ }
+ return options;
+ }
+};
+
+test "lspbench options reject unknown flags and multiple roots before probing" {
+ const defaults = try Options.parse(&.{});
+ try std.testing.expect(!defaults.json and !defaults.check);
+ try std.testing.expectEqualStrings(".", defaults.root);
+ for ([_][]const []const u8{
+ &.{ "--json", "repo root", "--check" },
+ &.{ "repo root", "--check", "--json" },
+ }) |args| {
+ const options = try Options.parse(args);
+ try std.testing.expect(options.json and options.check);
+ try std.testing.expectEqualStrings("repo root", options.root);
+ }
+ for ([_][]const []const u8{
+ &.{ "--reps", "3" },
+ &.{ "--json", "--unknown" },
+ &.{ "repo", "--check=true" },
+ }) |args| try std.testing.expectError(error.UnknownArgument, Options.parse(args));
+ try std.testing.expectError(error.MultipleRoots, Options.parse(&.{ "first", "second" }));
+ try std.testing.expectError(error.MultipleRoots, Options.parse(&.{ "first", "--check", "second" }));
+}
+
pub fn main(init: std.process.Init) !void {
- // page_allocator like the other harnesses: this measures the BACKEND, and
- // a debug allocator's bookkeeping would land in every number.
const gpa = std.heap.page_allocator;
const args = try init.minimal.args.toSlice(init.arena.allocator());
- var json = false;
- var root: []const u8 = ".";
- for (args[1..]) |a| {
- if (std.mem.eql(u8, a, "--json")) json = true else root = a;
- }
+ const options = try Options.parse(args[1..]);
const rss0 = rssKib();
var results: [anchors.len]Result = undefined;
var result_count: usize = 0;
for (anchors) |an| {
- const path = try std.fs.path.join(gpa, &.{ root, an.file });
- defer gpa.free(path);
- const src = readZ(gpa, path) catch continue;
- defer gpa.free(src);
- const off = std.mem.indexOf(u8, src, an.needle) orelse {
- std.debug.print("bench: anchor not found in {s}: `{s}`\n", .{ an.file, an.needle });
+ results[result_count] = probe(gpa, options.root, an) catch |err| {
+ std.debug.print("bench: {s} {s} `{s}`: {s}\n", .{ an.file, @tagName(an.kind), an.needle, @errorName(err) });
continue;
};
- var realbuf: [4096]u8 = undefined;
- var pz: [4096:0]u8 = undefined;
- const pz_s = std.fmt.bufPrintSentinel(&pz, "{s}", .{path}, 0) catch continue;
- const abs = std.mem.span(realpath(pz_s.ptr, &realbuf) orelse continue);
- const dir = std.fs.path.dirname(abs) orelse "/";
- const req: lsp.Req = .{
- .kind = an.kind,
- .path = abs,
- .source = src,
- .offset = @intCast(off + an.at),
- .arg = if (an.kind == .rename) "renamed_by_bench" else if (an.kind == .workspace_symbols) "Kind" else "",
- .root = dir,
- };
-
- var r: Result = .{ .kind = an.kind };
- // COLD: whatever the backend has to build the first time counts.
- r.cold_us = runOnce(gpa, req, &r);
- // WARM: median of warm_iters, so one scheduling hiccup cannot flatter
- // or damn a backend.
- var samples: [warm_iters]u64 = undefined;
- for (&samples) |*s| {
- var throwaway: Result = .{ .kind = an.kind };
- s.* = runOnce(gpa, req, &throwaway);
- }
- std.mem.sort(u64, &samples, {}, std.sort.asc(u64));
- r.warm_us = samples[warm_iters / 2];
- r.correct = !r.empty and (an.expect.len == 0 or blk: {
- var out: std.Io.Writer.Allocating = .init(gpa);
- defer out.deinit();
- var arena: std.heap.ArenaAllocator = .init(gpa);
- defer arena.deinit();
- lsp.query(gpa, arena.allocator(), req, &out.writer);
- break :blk std.mem.indexOf(u8, out.written(), an.expect) != null;
- });
- results[result_count] = r;
result_count += 1;
}
const rss = rssKib() -| rss0;
- if (json) reportJson(results[0..result_count], rss) else reportText(results[0..result_count], rss);
+ if (options.json) reportJson(results[0..result_count], rss) else reportText(results[0..result_count], rss);
+ if (options.check) try checkResults(results[0..result_count], anchors.len);
+}
+
+fn probe(gpa: std.mem.Allocator, root: []const u8, an: Anchor) !Result {
+ const path = try std.fs.path.join(gpa, &.{ root, an.file });
+ defer gpa.free(path);
+ const src = try readZ(gpa, path);
+ defer gpa.free(src);
+ const off = std.mem.indexOf(u8, src, an.needle) orelse return error.MissingAnchor;
+ if (an.at > an.needle.len) return error.InvalidAnchor;
+ var realbuf: [4096]u8 = undefined;
+ var pz: [4096:0]u8 = undefined;
+ const pz_s = try std.fmt.bufPrintSentinel(&pz, "{s}", .{path}, 0);
+ const abs = std.mem.span(realpath(pz_s.ptr, &realbuf) orelse return error.RealPathFailed);
+ const req: lsp.Req = .{
+ .kind = an.kind,
+ .path = abs,
+ .source = src,
+ .offset = @intCast(off + an.at),
+ .arg = if (an.kind == .rename) "renamed_by_bench" else if (an.kind == .workspace_symbols) "Kind" else "",
+ .root = std.fs.path.dirname(abs) orelse "/",
+ };
+ var r: Result = .{ .kind = an.kind };
+ r.cold_us = try runOnce(gpa, req, an.expect, &r);
+ var samples: [warm_iters]u64 = undefined;
+ for (&samples) |*sample| {
+ var warm: Result = .{ .kind = an.kind };
+ sample.* = try runOnce(gpa, req, an.expect, &warm);
+ r.correct = r.correct and warm.correct;
+ }
+ std.mem.sort(u64, &samples, {}, std.sort.asc(u64));
+ r.warm_us = samples[warm_iters / 2];
+ return r;
+}
+
+fn checkResults(results: []const Result, required: usize) !void {
+ if (results.len != required) return error.MissingProbes;
+ for (results) |r| if (r.empty or !r.correct) return error.IncorrectProbe;
}
-fn runOnce(gpa: std.mem.Allocator, req: lsp.Req, r: *Result) u64 {
+fn runOnce(gpa: std.mem.Allocator, req: lsp.Req, expected: []const u8, r: *Result) !u64 {
var arena: std.heap.ArenaAllocator = .init(gpa);
defer arena.deinit();
var out: std.Io.Writer.Allocating = .init(gpa);
defer out.deinit();
const t0 = nowNs();
- lsp.query(gpa, arena.allocator(), req, &out.writer);
+ try lsp.query(gpa, arena.allocator(), req, &out.writer);
const ns = nowNs() -| t0;
- if (out.written().len > 0) {
- r.empty = false;
- r.rows = std.mem.count(u8, out.written(), "\n");
- }
+ r.accept(out.written(), expected);
return ns / 1000;
}
+test "lspbench check rejects omitted empty and incorrect probes" {
+ var result: Result = .{ .kind = .definition };
+ try std.testing.expectError(error.MissingProbes, checkResults(&.{}, 1));
+ try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1));
+ result.accept("actual.zig:1:1: symbol\n", "expected.zig");
+ try std.testing.expect(!result.empty);
+ try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1));
+ result.accept("actual.zig:1:1: symbol\n", "actual.zig");
+ try checkResults(&.{result}, 1);
+ try std.testing.expectError(error.MissingProbes, checkResults(&.{result}, 2));
+ result.accept("", "");
+ try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1));
+}
+
+test "lspbench reports missing fixture and anchor before querying a backend" {
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var path_buf: [4096]u8 = undefined;
+ const root = path_buf[0..try tmp.dir.realPath(std.testing.io, &path_buf)];
+ const anchor: Anchor = .{ .file = "source.zig", .needle = "pub const value", .kind = .definition };
+ try std.testing.expectError(error.OpenFailed, probe(std.testing.allocator, root, anchor));
+ try tmp.dir.writeFile(std.testing.io, .{ .sub_path = anchor.file, .data = "const value = 1;\n" });
+ try std.testing.expectError(error.MissingAnchor, probe(std.testing.allocator, root, anchor));
+ var invalid = anchor;
+ invalid.needle = "value";
+ invalid.at = 6;
+ try std.testing.expectError(error.InvalidAnchor, probe(std.testing.allocator, root, invalid));
+}
+
fn reportText(rs: []const Result, rss: u64) void {
const o = std.debug.print;
o("backend: {s}\n\n", .{lsp.backend_name});
@@ -284,8 +280,6 @@ fn readZ(gpa: std.mem.Allocator, path: []const u8) ![:0]u8 {
return buf[0..len :0];
}
-/// VmHWM from /proc/self/status — the peak, not the current, so a backend that
-/// frees its index before returning still pays for having built it.
fn rssKib() u64 {
const fd = libc.open("/proc/self/status", .{ .ACCMODE = .RDONLY });
if (fd < 0) return 0;
generated by cgit v1.3 (git 2.53.0) at 2026-10-02 19:44:47 +0000