path: root/test/lspbench.zig

//! The language-backend scoreboard: one binary, three numbers.
//!
//!   zig build lspbench            -- latency + feature matrix over pardes's own src/
//!   zig build lspbench -- --json  -- the same, machine-readable
//!
//! Every competing backend links the same harness against the same corpus and
//! the same query list, so the columns mean the same thing across all of them.
//! What it measures, in the order the evaluation weighs it:
//!
//!   FEATURES  which lsp.Kind values actually return rows. Claiming support in
//!             `lsp.supports` and returning nothing is a FALSE claim and shows
//!             up as `claimed-empty` — the harness trusts results, not flags.
//!   LATENCY   cold (first query, index build included) and warm (median of N)
//!             per kind. Cold is what a keypress costs the first time; warm is
//!             what it costs forever after. Both matter and they differ by
//!             orders of magnitude for an indexing backend.
//!   MEMORY    peak RSS delta across the run, read from /proc.
//!
//! Lines of code is not measured here — it is `jj diff --stat` against the
//! base commit, which is the honest number (a backend that vendors a library
//! pays for what it vendors only in build time, not in code we maintain).
const std = @import("std");
const libc = std.c;
const lsp = @import("pardes").lsp;

// 0.16 slimmed std.fs (no cwd(), no realpathAlloc) and this repo goes through
// libc everywhere for exactly that reason — see look.zig.
extern "c" fn realpath(path: [*:0]const u8, resolved: [*]u8) ?[*:0]u8;

/// std.time.Timer is gone in 0.16; clock_gettime is what dump.zig already uses.
fn nowNs() u64 {
    var ts: std.c.timespec = undefined;
    _ = std.c.clock_gettime(.MONOTONIC, &ts);
    return @as(u64, @intCast(ts.sec)) *| 1_000_000_000 +| @as(u64, @intCast(ts.nsec));
}

pub const std_options: std.Options = .{ .log_level = .err };

/// One probe. The corpus is pardes's own source: real Zig, in this repo, that
/// every implementation can reach without a fixture tree to keep in sync.
///
/// The cursor is pinned by SYMBOL, not by line — the needle is searched for at
/// startup — so editing pardes.zig cannot silently rot the bench into probing
/// a blank line and calling the result "no support".
///
/// `expect` is a substring the rows must contain for the probe to count as
/// CORRECT rather than merely non-empty. A backend that returns a plausible
/// wrong location scores worse than one that returns nothing, and only this
/// field can tell those two apart.
const Anchor = struct {
    file: []const u8,
    /// the exact source text to put the cursor on; the FIRST occurrence wins
    needle: []const u8,
    /// byte offset into `needle` where the cursor sits
    at: u32 = 0,
    kind: lsp.Kind,
    expect: []const u8 = "",
};

const anchors = [_]Anchor{
    // a call to a function defined in the same file
    .{ .file = "src/pardes.zig", .needle = "modal.hxLineCount(text)", .at = 6, .kind = .definition, .expect = "modal.zig" },
    // a plain local/decl reference within one file
    .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .definition, .expect = "lsp.zig" },
    // a std reference: needs the zig lib dir, which is the first thing a
    // single-file backend cannot do
    .{ .file = "src/lsp/lsp.zig", .needle = "std.mem.count(u8", .at = 8, .kind = .definition, .expect = "mem.zig" },
    // hover over the same symbol
    .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .hover, .expect = "" },
    // the file's own symbols
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .document_symbols, .expect = "Kind" },
    .{ .file = "src/modal.zig", .needle = "pub fn ", .at = 7, .kind = .document_symbols, .expect = "" },
    // references to a symbol used in several places
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .references, .expect = "" },
    // Diagnostics and format probe a DELIBERATELY BROKEN fixture, never the
    // real source. On a clean corpus the correct answer is nothing, which is
    // byte-identical to "this backend has no diagnostics" — a blind spot that
    // rewards emitting a filler row and punishes honesty. test/lspfixture has
    // an unused local (semantic: needs a compiler front end, not a tokenizer)
    // and a misformatted fn (needs a formatter), so both probes have real work.
    .{ .file = "test/lspfixture/broken.zig", .needle = "unused_local", .at = 0, .kind = .diagnostics, .expect = "broken.zig" },
    // the remaining kinds, probed once each so the matrix is complete
    .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .declaration, .expect = "" },
    // cursor on `out`, whose type is `*std.Io.Writer` — a type_definition probe
    // wants a NAME whose type must be resolved, not the type spelled out
    .{ .file = "src/lsp/lsp.zig", .needle = "out: *std.Io.Writer", .at = 0, .kind = .type_definition, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .implementation, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .select_refs, .expect = "" },
    .{ .file = "test/lspfixture/broken.zig", .needle = "badly_spaced", .at = 0, .kind = .format, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .code_action, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .rename, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .workspace_symbols, .expect = "" },
    .{ .file = "test/lspfixture/broken.zig", .needle = "unused_local", .at = 0, .kind = .workspace_diagnostics, .expect = "broken.zig" },
};

const Result = struct {
    kind: lsp.Kind,
    /// no rows came back
    empty: bool = true,
    /// rows came back AND contained `expect` (or nothing was expected)
    correct: bool = false,
    rows: usize = 0,
    cold_us: u64 = 0,
    warm_us: u64 = 0,
};

const warm_iters = 20;

pub fn main(init: std.process.Init) !void {
    // page_allocator like the other harnesses: this measures the BACKEND, and
    // a debug allocator's bookkeeping would land in every number.
    const gpa = std.heap.page_allocator;

    const args = try init.minimal.args.toSlice(init.arena.allocator());
    var json = false;
    var root: []const u8 = ".";
    for (args[1..]) |a| {
        if (std.mem.eql(u8, a, "--json")) json = true else root = a;
    }

    const rss0 = rssKib();
    var results: std.ArrayList(Result) = .empty;
    defer results.deinit(gpa);

    for (anchors) |an| {
        const path = try std.fs.path.join(gpa, &.{ root, an.file });
        defer gpa.free(path);
        const src = readZ(gpa, path) catch continue;
        defer gpa.free(src);
        const off = std.mem.indexOf(u8, src, an.needle) orelse {
            std.debug.print("bench: anchor not found in {s}: `{s}`\n", .{ an.file, an.needle });
            continue;
        };
        var realbuf: [4096]u8 = undefined;
        var pz: [4096:0]u8 = undefined;
        const pz_s = std.fmt.bufPrintSentinel(&pz, "{s}", .{path}, 0) catch continue;
        const abs = std.mem.span(realpath(pz_s.ptr, &realbuf) orelse continue);
        const dir = std.fs.path.dirname(abs) orelse "/";
        const req: lsp.Req = .{
            .kind = an.kind,
            .path = abs,
            .source = src,
            .offset = @intCast(off + an.at),
            .arg = if (an.kind == .rename) "renamed_by_bench" else if (an.kind == .workspace_symbols) "Kind" else "",
            .root = dir,
        };

        var r: Result = .{ .kind = an.kind };
        // COLD: whatever the backend has to build the first time counts.
        r.cold_us = runOnce(gpa, req, &r);
        // WARM: median of warm_iters, so one scheduling hiccup cannot flatter
        // or damn a backend.
        var samples: [warm_iters]u64 = undefined;
        for (&samples) |*s| {
            var throwaway: Result = .{ .kind = an.kind };
            s.* = runOnce(gpa, req, &throwaway);
        }
        std.mem.sort(u64, &samples, {}, std.sort.asc(u64));
        r.warm_us = samples[warm_iters / 2];
        r.correct = !r.empty and (an.expect.len == 0 or blk: {
            var out: std.Io.Writer.Allocating = .init(gpa);
            defer out.deinit();
            var arena: std.heap.ArenaAllocator = .init(gpa);
            defer arena.deinit();
            lsp.query(gpa, arena.allocator(), req, &out.writer);
            break :blk std.mem.indexOf(u8, out.written(), an.expect) != null;
        });
        try results.append(gpa, r);
    }

    const rss = rssKib() -| rss0;
    if (json) reportJson(results.items, rss) else reportText(results.items, rss);
}

fn runOnce(gpa: std.mem.Allocator, req: lsp.Req, r: *Result) u64 {
    var arena: std.heap.ArenaAllocator = .init(gpa);
    defer arena.deinit();
    var out: std.Io.Writer.Allocating = .init(gpa);
    defer out.deinit();
    const t0 = nowNs();
    lsp.query(gpa, arena.allocator(), req, &out.writer);
    const ns = nowNs() -| t0;
    if (out.written().len > 0) {
        r.empty = false;
        r.rows = std.mem.count(u8, out.written(), "\n");
    }
    return ns / 1000;
}

fn reportText(rs: []const Result, rss: u64) void {
    const o = std.debug.print;
    o("backend: {s}\n\n", .{lsp.backend_name});
    o("{s:<24} {s:>9} {s:>7} {s:>10} {s:>10}  {s}\n", .{ "kind", "supports", "rows", "cold(us)", "warm(us)", "verdict" });
    o("{s}\n", .{"-" ** 78});
    var claimed: usize = 0;
    var working: usize = 0;
    var lies: usize = 0;
    for (rs) |r| {
        const sup = lsp.supports.contains(r.kind);
        if (sup) claimed += 1;
        const verdict: []const u8 = if (!sup and r.empty)
            "-"
        else if (sup and r.empty)
            "CLAIMED-EMPTY"
        else if (!sup and !r.empty)
            "unclaimed-works"
        else if (r.correct)
            "ok"
        else
            "rows-but-wrong";
        if (sup and !r.empty and r.correct) working += 1;
        if (sup and r.empty) lies += 1;
        o("{s:<24} {s:>9} {d:>7} {d:>10} {d:>10}  {s}\n", .{
            @tagName(r.kind), if (sup) "yes" else "no", r.rows, r.cold_us, r.warm_us, verdict,
        });
    }
    o("\nclaimed {d}, verified working {d}, false claims {d}\n", .{ claimed, working, lies });
    o("peak rss delta: {d} KiB\n", .{rss});
}

fn reportJson(rs: []const Result, rss: u64) void {
    const o = std.debug.print;
    o("{{\"backend\":\"{s}\",\"rss_kib\":{d},\"probes\":[", .{ lsp.backend_name, rss });
    for (rs, 0..) |r, i| {
        if (i > 0) o(",", .{});
        o("{{\"kind\":\"{s}\",\"supports\":{},\"rows\":{d},\"correct\":{},\"cold_us\":{d},\"warm_us\":{d}}}", .{
            @tagName(r.kind), lsp.supports.contains(r.kind), r.rows, r.correct, r.cold_us, r.warm_us,
        });
    }
    o("]}}\n", .{});
}

fn readZ(gpa: std.mem.Allocator, path: []const u8) ![:0]u8 {
    var pathbuf: [4096]u8 = undefined;
    const path_z = try std.fmt.bufPrintSentinel(&pathbuf, "{s}", .{path}, 0);
    const fd = libc.open(path_z, .{ .ACCMODE = .RDONLY });
    if (fd < 0) return error.OpenFailed;
    defer _ = libc.close(fd);
    var buf: std.ArrayList(u8) = .empty;
    errdefer buf.deinit(gpa);
    var chunk: [16384]u8 = undefined;
    while (true) {
        const n = libc.read(fd, &chunk, chunk.len);
        if (n < 0) {
            if (libc.errno(n) == .INTR) continue;
            return error.ReadFailed;
        }
        if (n == 0) break;
        try buf.appendSlice(gpa, chunk[0..@intCast(n)]);
    }
    return buf.toOwnedSliceSentinel(gpa, 0);
}

/// VmHWM from /proc/self/status — the peak, not the current, so a backend that
/// frees its index before returning still pays for having built it.
fn rssKib() u64 {
    const fd = libc.open("/proc/self/status", .{ .ACCMODE = .RDONLY });
    if (fd < 0) return 0;
    defer _ = libc.close(fd);
    var buf: [4096]u8 = undefined;
    const n = libc.read(fd, &buf, buf.len);
    if (n <= 0) return 0;
    var it = std.mem.tokenizeScalar(u8, buf[0..@intCast(n)], '\n');
    while (it.next()) |line| {
        if (!std.mem.startsWith(u8, line, "VmHWM:")) continue;
        var t = std.mem.tokenizeAny(u8, line["VmHWM:".len..], " \t");
        const v = t.next() orelse return 0;
        return std.fmt.parseInt(u64, v, 10) catch 0;
    }
    return 0;
}