const std = @import("std"); const libc = std.c; const lsp = @import("pardes").lsp; extern "c" fn realpath(path: [*:0]const u8, resolved: [*]u8) ?[*:0]u8; fn nowNs() u64 { var ts: std.c.timespec = undefined; _ = std.c.clock_gettime(.MONOTONIC, &ts); return @as(u64, @intCast(ts.sec)) *| 1_000_000_000 +| @as(u64, @intCast(ts.nsec)); } pub const std_options: std.Options = .{ .log_level = .err }; // Coordinates are byte offsets within the first needle occurrence, not source line numbers. const Anchor = struct { file: []const u8, needle: []const u8, at: u32 = 0, kind: lsp.Kind, expect: []const u8 = "", }; const anchors = [_]Anchor{ .{ .file = "src/pardes.zig", .needle = "modal.cursorLineCount(text)", .at = 6, .kind = .definition, .expect = "modal.zig" }, .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .definition, .expect = "lsp.zig" }, .{ .file = "src/lsp/lsp.zig", .needle = "std.mem.count(u8", .at = 8, .kind = .definition, .expect = "mem.zig" }, .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .hover, .expect = "" }, .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .document_symbols, .expect = "Kind" }, .{ .file = "src/modal.zig", .needle = "pub fn ", .at = 7, .kind = .document_symbols, .expect = "" }, .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .references, .expect = "" }, .{ .file = "test/lspfixture/broken.zig", .needle = "unused_local", .at = 0, .kind = .diagnostics, .expect = "broken.zig" }, .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .declaration, .expect = "" }, .{ .file = "src/lsp/lsp.zig", .needle = "out: *std.Io.Writer", .at = 0, .kind = .type_definition, .expect = "" }, .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .implementation, .expect = "" }, .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .select_refs, .expect = "" }, .{ .file = "test/lspfixture/broken.zig", .needle = "badly_spaced", .at = 0, .kind = .format, .expect = "" }, .{ .file = "src/lsp/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .code_action, .expect = "" }, .{ .file = "src/lsp/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .rename, .expect = "" }, .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .workspace_symbols, .expect = "" }, .{ .file = "test/lspfixture/broken.zig", .needle = "unused_local", .at = 0, .kind = .workspace_diagnostics, .expect = "broken.zig" }, .{ .file = "src/lsp/lsp.zig", .needle = "std.mem.count(u8", .at = 4, .kind = .completion, .expect = "std.zig" }, .{ .file = "src/lsp/lsp_zls.zig", .needle = ".declaration => try rowForToken", .at = 1, .kind = .completion, .expect = "type_definition" }, .{ .file = "src/pardes.zig", .needle = ".definition => .definition,", .at = 1, .kind = .completion, .expect = "" }, // The repeated name proves the result includes the declaration, not just the cursor line. .{ .file = "test/lspfixture/dotcomplete.zig", .needle = "return switch (s) {\n .", .at = 29, .kind = .completion, .expect = "verdigris verdigris," }, // Indentation keeps the fixture header from becoming the first needle match. .{ .file = "test/lspfixture/dothalf.zig", .needle = " const z: Shade = .", .at = 22, .kind = .completion, .expect = "verdigris" }, }; const Result = struct { kind: lsp.Kind, empty: bool = true, correct: bool = false, rows: usize = 0, cold_us: u64 = 0, warm_us: u64 = 0, fn accept(r: *Result, output: []const u8, expected: []const u8) void { r.empty = output.len == 0; r.rows = std.mem.count(u8, output, "\n"); r.correct = !r.empty and (expected.len == 0 or std.mem.indexOf(u8, output, expected) != null); } }; const warm_iters = 20; const Options = struct { json: bool = false, check: bool = false, root: []const u8 = ".", fn parse(args: []const []const u8) !Options { var options: Options = .{}; var has_root = false; for (args) |arg| { if (std.mem.eql(u8, arg, "--json")) { options.json = true; } else if (std.mem.eql(u8, arg, "--check")) { options.check = true; } else if (std.mem.startsWith(u8, arg, "-")) { return error.UnknownArgument; } else { if (has_root) return error.MultipleRoots; options.root = arg; has_root = true; } } return options; } }; test "lspbench options reject unknown flags and multiple roots before probing" { const defaults = try Options.parse(&.{}); try std.testing.expect(!defaults.json and !defaults.check); try std.testing.expectEqualStrings(".", defaults.root); for ([_][]const []const u8{ &.{ "--json", "repo root", "--check" }, &.{ "repo root", "--check", "--json" }, }) |args| { const options = try Options.parse(args); try std.testing.expect(options.json and options.check); try std.testing.expectEqualStrings("repo root", options.root); } for ([_][]const []const u8{ &.{ "--reps", "3" }, &.{ "--json", "--unknown" }, &.{ "repo", "--check=true" }, }) |args| try std.testing.expectError(error.UnknownArgument, Options.parse(args)); try std.testing.expectError(error.MultipleRoots, Options.parse(&.{ "first", "second" })); try std.testing.expectError(error.MultipleRoots, Options.parse(&.{ "first", "--check", "second" })); } pub fn main(init: std.process.Init) !void { const gpa = std.heap.page_allocator; const args = try init.minimal.args.toSlice(init.arena.allocator()); const options = try Options.parse(args[1..]); const rss0 = rssKib(); var results: [anchors.len]Result = undefined; var result_count: usize = 0; for (anchors) |an| { results[result_count] = probe(gpa, options.root, an) catch |err| { std.debug.print("bench: {s} {s} `{s}`: {s}\n", .{ an.file, @tagName(an.kind), an.needle, @errorName(err) }); continue; }; result_count += 1; } const rss = rssKib() -| rss0; if (options.json) reportJson(results[0..result_count], rss) else reportText(results[0..result_count], rss); if (options.check) try checkResults(results[0..result_count], anchors.len); } fn probe(gpa: std.mem.Allocator, root: []const u8, an: Anchor) !Result { const path = try std.fs.path.join(gpa, &.{ root, an.file }); defer gpa.free(path); const src = try readZ(gpa, path); defer gpa.free(src); const off = std.mem.indexOf(u8, src, an.needle) orelse return error.MissingAnchor; if (an.at > an.needle.len) return error.InvalidAnchor; var realbuf: [4096]u8 = undefined; var pz: [4096:0]u8 = undefined; const pz_s = try std.fmt.bufPrintSentinel(&pz, "{s}", .{path}, 0); const abs = std.mem.span(realpath(pz_s.ptr, &realbuf) orelse return error.RealPathFailed); const req: lsp.Req = .{ .kind = an.kind, .path = abs, .source = src, .offset = @intCast(off + an.at), .arg = if (an.kind == .rename) "renamed_by_bench" else if (an.kind == .workspace_symbols) "Kind" else "", .root = std.fs.path.dirname(abs) orelse "/", }; var r: Result = .{ .kind = an.kind }; r.cold_us = try runOnce(gpa, req, an.expect, &r); var samples: [warm_iters]u64 = undefined; for (&samples) |*sample| { var warm: Result = .{ .kind = an.kind }; sample.* = try runOnce(gpa, req, an.expect, &warm); r.correct = r.correct and warm.correct; } std.mem.sort(u64, &samples, {}, std.sort.asc(u64)); r.warm_us = samples[warm_iters / 2]; return r; } fn checkResults(results: []const Result, required: usize) !void { if (results.len != required) return error.MissingProbes; for (results) |r| if (r.empty or !r.correct) return error.IncorrectProbe; } fn runOnce(gpa: std.mem.Allocator, req: lsp.Req, expected: []const u8, r: *Result) !u64 { var arena: std.heap.ArenaAllocator = .init(gpa); defer arena.deinit(); var out: std.Io.Writer.Allocating = .init(gpa); defer out.deinit(); const t0 = nowNs(); try lsp.query(gpa, arena.allocator(), req, &out.writer); const ns = nowNs() -| t0; r.accept(out.written(), expected); return ns / 1000; } test "lspbench check rejects omitted empty and incorrect probes" { var result: Result = .{ .kind = .definition }; try std.testing.expectError(error.MissingProbes, checkResults(&.{}, 1)); try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1)); result.accept("actual.zig:1:1: symbol\n", "expected.zig"); try std.testing.expect(!result.empty); try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1)); result.accept("actual.zig:1:1: symbol\n", "actual.zig"); try checkResults(&.{result}, 1); try std.testing.expectError(error.MissingProbes, checkResults(&.{result}, 2)); result.accept("", ""); try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1)); } test "lspbench reports missing fixture and anchor before querying a backend" { var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var path_buf: [4096]u8 = undefined; const root = path_buf[0..try tmp.dir.realPath(std.testing.io, &path_buf)]; const anchor: Anchor = .{ .file = "source.zig", .needle = "pub const value", .kind = .definition }; try std.testing.expectError(error.OpenFailed, probe(std.testing.allocator, root, anchor)); try tmp.dir.writeFile(std.testing.io, .{ .sub_path = anchor.file, .data = "const value = 1;\n" }); try std.testing.expectError(error.MissingAnchor, probe(std.testing.allocator, root, anchor)); var invalid = anchor; invalid.needle = "value"; invalid.at = 6; try std.testing.expectError(error.InvalidAnchor, probe(std.testing.allocator, root, invalid)); } fn reportText(rs: []const Result, rss: u64) void { const o = std.debug.print; o("backend: {s}\n\n", .{lsp.backend_name}); o("{s:<24} {s:>9} {s:>7} {s:>10} {s:>10} {s}\n", .{ "kind", "supports", "rows", "cold(us)", "warm(us)", "verdict" }); o("{s}\n", .{"-" ** 78}); var claimed: usize = 0; var working: usize = 0; var lies: usize = 0; for (rs) |r| { const sup = lsp.supports.contains(r.kind); if (sup) claimed += 1; const verdict: []const u8 = if (!sup and r.empty) "-" else if (sup and r.empty) "CLAIMED-EMPTY" else if (!sup and !r.empty) "unclaimed-works" else if (r.correct) "ok" else "rows-but-wrong"; if (sup and !r.empty and r.correct) working += 1; if (sup and r.empty) lies += 1; o("{s:<24} {s:>9} {d:>7} {d:>10} {d:>10} {s}\n", .{ @tagName(r.kind), if (sup) "yes" else "no", r.rows, r.cold_us, r.warm_us, verdict, }); } o("\nclaimed {d}, verified working {d}, false claims {d}\n", .{ claimed, working, lies }); o("peak rss delta: {d} KiB\n", .{rss}); } fn reportJson(rs: []const Result, rss: u64) void { const o = std.debug.print; o("{{\"backend\":\"{s}\",\"rss_kib\":{d},\"probes\":[", .{ lsp.backend_name, rss }); for (rs, 0..) |r, i| { if (i > 0) o(",", .{}); o("{{\"kind\":\"{s}\",\"supports\":{},\"rows\":{d},\"correct\":{},\"cold_us\":{d},\"warm_us\":{d}}}", .{ @tagName(r.kind), lsp.supports.contains(r.kind), r.rows, r.correct, r.cold_us, r.warm_us, }); } o("]}}\n", .{}); } fn readZ(gpa: std.mem.Allocator, path: []const u8) ![:0]u8 { var pathbuf: [4096]u8 = undefined; const path_z = try std.fmt.bufPrintSentinel(&pathbuf, "{s}", .{path}, 0); const fd = libc.open(path_z, .{ .ACCMODE = .RDONLY }); if (fd < 0) return error.OpenFailed; defer _ = libc.close(fd); const end = libc.lseek(fd, 0, libc.SEEK.END); if (end < 0 or libc.lseek(fd, 0, libc.SEEK.SET) < 0) return error.ReadFailed; const len: usize = @intCast(end); const buf = try gpa.alloc(u8, len + 1); errdefer gpa.free(buf); var filled: usize = 0; while (filled < len) { const n = libc.read(fd, buf[filled..].ptr, len - filled); if (n < 0) { if (libc.errno(n) == .INTR) continue; return error.ReadFailed; } if (n == 0) return error.UnexpectedEof; filled += @intCast(n); } buf[len] = 0; return buf[0..len :0]; } fn rssKib() u64 { const fd = libc.open("/proc/self/status", .{ .ACCMODE = .RDONLY }); if (fd < 0) return 0; defer _ = libc.close(fd); var buf: [4096]u8 = undefined; const n = libc.read(fd, &buf, buf.len); if (n <= 0) return 0; var it = std.mem.tokenizeScalar(u8, buf[0..@intCast(n)], '\n'); while (it.next()) |line| { if (!std.mem.startsWith(u8, line, "VmHWM:")) continue; var t = std.mem.tokenizeAny(u8, line["VmHWM:".len..], " \t"); const v = t.next() orelse return 0; return std.fmt.parseInt(u64, v, 10) catch 0; } return 0; }