path: root/test/lspbench.zig

const std = @import("std");
const libc = std.c;
const lsp = @import("pardes").lsp;

extern "c" fn realpath(path: [*:0]const u8, resolved: [*]u8) ?[*:0]u8;

fn nowNs() u64 {
    var ts: std.c.timespec = undefined;
    _ = std.c.clock_gettime(.MONOTONIC, &ts);
    return @as(u64, @intCast(ts.sec)) *| 1_000_000_000 +| @as(u64, @intCast(ts.nsec));
}

pub const std_options: std.Options = .{ .log_level = .err };

// Coordinates are byte offsets within the first needle occurrence, not source line numbers.
const Anchor = struct {
    file: []const u8,
    needle: []const u8,
    at: u32 = 0,
    kind: lsp.Kind,
    expect: []const u8 = "",
};

const anchors = [_]Anchor{
    .{ .file = "src/edit.zig", .needle = "modal.cursorLineCount(text)", .at = 6, .kind = .definition, .expect = "modal.zig" },
    .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .definition, .expect = "lsp.zig" },
    .{ .file = "src/lsp/lsp.zig", .needle = "std.mem.count(u8", .at = 8, .kind = .definition, .expect = "mem.zig" },
    .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .hover, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .document_symbols, .expect = "Kind" },
    .{ .file = "src/modal.zig", .needle = "pub fn ", .at = 7, .kind = .document_symbols, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .references, .expect = "" },
    .{ .file = "test/lspfixture/broken.zig", .needle = "unused_local", .at = 0, .kind = .diagnostics, .expect = "broken.zig" },
    .{ .file = "src/lsp/lsp.zig", .needle = "lineCol(source", .at = 0, .kind = .declaration, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "out: *std.Io.Writer", .at = 0, .kind = .type_definition, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .implementation, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .select_refs, .expect = "" },
    .{ .file = "test/lspfixture/broken.zig", .needle = "badly_spaced", .at = 0, .kind = .format, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .code_action, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub fn query", .at = 7, .kind = .rename, .expect = "" },
    .{ .file = "src/lsp/lsp.zig", .needle = "pub const Kind", .at = 11, .kind = .workspace_symbols, .expect = "" },
    .{ .file = "test/lspfixture/broken.zig", .needle = "unused_local", .at = 0, .kind = .workspace_diagnostics, .expect = "broken.zig" },
    .{ .file = "src/lsp/lsp.zig", .needle = "std.mem.count(u8", .at = 4, .kind = .completion, .expect = "std.zig" },
    .{ .file = "src/normal.zig", .needle = ".definition => .definition,", .at = 1, .kind = .completion, .expect = "" },
    // The repeated name proves the result includes the declaration, not just the cursor line.
    .{ .file = "test/lspfixture/dotcomplete.zig", .needle = "return switch (s) {\n        .", .at = 29, .kind = .completion, .expect = "verdigris  verdigris," },
    // Indentation keeps the fixture header from becoming the first needle match.
    .{ .file = "test/lspfixture/dothalf.zig", .needle = "    const z: Shade = .", .at = 22, .kind = .completion, .expect = "verdigris" },
};

const Result = struct {
    kind: lsp.Kind,
    empty: bool = true,
    correct: bool = false,
    rows: usize = 0,
    cold_us: u64 = 0,
    warm_us: u64 = 0,

    fn accept(r: *Result, output: []const u8, expected: []const u8) void {
        r.empty = output.len == 0;
        r.rows = std.mem.count(u8, output, "\n");
        r.correct = !r.empty and (expected.len == 0 or std.mem.indexOf(u8, output, expected) != null);
    }
};

const warm_iters = 20;

const Options = struct {
    json: bool = false,
    check: bool = false,
    root: []const u8 = ".",

    fn parse(args: []const []const u8) !Options {
        var options: Options = .{};
        var has_root = false;
        for (args) |arg| {
            if (std.mem.eql(u8, arg, "--json")) {
                options.json = true;
            } else if (std.mem.eql(u8, arg, "--check")) {
                options.check = true;
            } else if (std.mem.startsWith(u8, arg, "-")) {
                return error.UnknownArgument;
            } else {
                if (has_root) return error.MultipleRoots;
                options.root = arg;
                has_root = true;
            }
        }
        return options;
    }
};

test "lspbench options reject unknown flags and multiple roots before probing" {
    const defaults = try Options.parse(&.{});
    try std.testing.expect(!defaults.json and !defaults.check);
    try std.testing.expectEqualStrings(".", defaults.root);
    for ([_][]const []const u8{
        &.{ "--json", "repo root", "--check" },
        &.{ "repo root", "--check", "--json" },
    }) |args| {
        const options = try Options.parse(args);
        try std.testing.expect(options.json and options.check);
        try std.testing.expectEqualStrings("repo root", options.root);
    }
    for ([_][]const []const u8{
        &.{ "--reps", "3" },
        &.{ "--json", "--unknown" },
        &.{ "repo", "--check=true" },
    }) |args| try std.testing.expectError(error.UnknownArgument, Options.parse(args));
    try std.testing.expectError(error.MultipleRoots, Options.parse(&.{ "first", "second" }));
    try std.testing.expectError(error.MultipleRoots, Options.parse(&.{ "first", "--check", "second" }));
}

pub fn main(init: std.process.Init) !void {
    const gpa = std.heap.page_allocator;

    const args = try init.minimal.args.toSlice(init.arena.allocator());
    if (args.len > 1 and std.mem.eql(u8, args[1], "probe")) return probeCommand(init);
    const options = try Options.parse(args[1..]);

    const rss0 = rssKib();
    var results: [anchors.len]Result = undefined;
    var result_count: usize = 0;

    for (anchors) |an| {
        results[result_count] = probe(gpa, options.root, an) catch |err| {
            std.debug.print("bench: {s} {s} `{s}`: {s}\n", .{ an.file, @tagName(an.kind), an.needle, @errorName(err) });
            continue;
        };
        result_count += 1;
    }

    const rss = rssKib() -| rss0;
    if (options.json) reportJson(results[0..result_count], rss) else reportText(results[0..result_count], rss);
    if (options.check) try checkResults(results[0..result_count], anchors.len);
}

fn probe(gpa: std.mem.Allocator, root: []const u8, an: Anchor) !Result {
    const path = try std.fs.path.join(gpa, &.{ root, an.file });
    defer gpa.free(path);
    const src = try readZ(gpa, path);
    defer gpa.free(src);
    const off = std.mem.indexOf(u8, src, an.needle) orelse return error.MissingAnchor;
    if (an.at > an.needle.len) return error.InvalidAnchor;
    var realbuf: [4096]u8 = undefined;
    var pz: [4096:0]u8 = undefined;
    const pz_s = try std.fmt.bufPrintSentinel(&pz, "{s}", .{path}, 0);
    const abs = std.mem.span(realpath(pz_s.ptr, &realbuf) orelse return error.RealPathFailed);
    const req: lsp.Req = .{
        .kind = an.kind,
        .path = abs,
        .source = src,
        .offset = @intCast(off + an.at),
        .arg = if (an.kind == .rename) "renamed_by_bench" else if (an.kind == .workspace_symbols) "Kind" else "",
        .root = std.fs.path.dirname(abs) orelse "/",
    };
    var r: Result = .{ .kind = an.kind };
    r.cold_us = try runOnce(gpa, req, an.expect, &r);
    var samples: [warm_iters]u64 = undefined;
    for (&samples) |*sample| {
        var warm: Result = .{ .kind = an.kind };
        sample.* = try runOnce(gpa, req, an.expect, &warm);
        r.correct = r.correct and warm.correct;
    }
    std.mem.sort(u64, &samples, {}, std.sort.asc(u64));
    r.warm_us = samples[warm_iters / 2];
    return r;
}

fn checkResults(results: []const Result, required: usize) !void {
    if (results.len != required) return error.MissingProbes;
    for (results) |r| if (r.empty or !r.correct) return error.IncorrectProbe;
}

fn runOnce(gpa: std.mem.Allocator, req: lsp.Req, expected: []const u8, r: *Result) !u64 {
    var arena: std.heap.ArenaAllocator = .init(gpa);
    defer arena.deinit();
    var out: std.Io.Writer.Allocating = .init(gpa);
    defer out.deinit();
    const t0 = nowNs();
    try lsp.query(gpa, arena.allocator(), req, &out.writer);
    const ns = nowNs() -| t0;
    r.accept(out.written(), expected);
    return ns / 1000;
}

test "lspbench check rejects omitted empty and incorrect probes" {
    var result: Result = .{ .kind = .definition };
    try std.testing.expectError(error.MissingProbes, checkResults(&.{}, 1));
    try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1));
    result.accept("actual.zig:1:1: symbol\n", "expected.zig");
    try std.testing.expect(!result.empty);
    try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1));
    result.accept("actual.zig:1:1: symbol\n", "actual.zig");
    try checkResults(&.{result}, 1);
    try std.testing.expectError(error.MissingProbes, checkResults(&.{result}, 2));
    result.accept("", "");
    try std.testing.expectError(error.IncorrectProbe, checkResults(&.{result}, 1));
}

test "lspbench reports missing fixture and anchor before querying a backend" {
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    var path_buf: [4096]u8 = undefined;
    const root = path_buf[0..try tmp.dir.realPath(std.testing.io, &path_buf)];
    const anchor: Anchor = .{ .file = "source.zig", .needle = "pub const value", .kind = .definition };
    try std.testing.expectError(error.OpenFailed, probe(std.testing.allocator, root, anchor));
    try tmp.dir.writeFile(std.testing.io, .{ .sub_path = anchor.file, .data = "const value = 1;\n" });
    try std.testing.expectError(error.MissingAnchor, probe(std.testing.allocator, root, anchor));
    var invalid = anchor;
    invalid.needle = "value";
    invalid.at = 6;
    try std.testing.expectError(error.InvalidAnchor, probe(std.testing.allocator, root, invalid));
}

fn reportText(rs: []const Result, rss: u64) void {
    const o = std.debug.print;
    o("backend: {s}\n\n", .{lsp.backend_name});
    o("{s:<24} {s:>9} {s:>7} {s:>10} {s:>10}  {s}\n", .{ "kind", "supports", "rows", "cold(us)", "warm(us)", "verdict" });
    o("{s}\n", .{"-" ** 78});
    var claimed: usize = 0;
    var working: usize = 0;
    var lies: usize = 0;
    for (rs) |r| {
        const sup = lsp.supports.contains(r.kind);
        if (sup) claimed += 1;
        const verdict: []const u8 = if (!sup and r.empty)
            "-"
        else if (sup and r.empty)
            "CLAIMED-EMPTY"
        else if (!sup and !r.empty)
            "unclaimed-works"
        else if (r.correct)
            "ok"
        else
            "rows-but-wrong";
        if (sup and !r.empty and r.correct) working += 1;
        if (sup and r.empty) lies += 1;
        o("{s:<24} {s:>9} {d:>7} {d:>10} {d:>10}  {s}\n", .{
            @tagName(r.kind), if (sup) "yes" else "no", r.rows, r.cold_us, r.warm_us, verdict,
        });
    }
    o("\nclaimed {d}, verified working {d}, false claims {d}\n", .{ claimed, working, lies });
    o("peak rss delta: {d} KiB\n", .{rss});
}

fn reportJson(rs: []const Result, rss: u64) void {
    const o = std.debug.print;
    o("{{\"backend\":\"{s}\",\"rss_kib\":{d},\"probes\":[", .{ lsp.backend_name, rss });
    for (rs, 0..) |r, i| {
        if (i > 0) o(",", .{});
        o("{{\"kind\":\"{s}\",\"supports\":{},\"rows\":{d},\"correct\":{},\"cold_us\":{d},\"warm_us\":{d}}}", .{
            @tagName(r.kind), lsp.supports.contains(r.kind), r.rows, r.correct, r.cold_us, r.warm_us,
        });
    }
    o("]}}\n", .{});
}

fn readZ(gpa: std.mem.Allocator, path: []const u8) ![:0]u8 {
    var pathbuf: [4096]u8 = undefined;
    const path_z = try std.fmt.bufPrintSentinel(&pathbuf, "{s}", .{path}, 0);
    const fd = libc.open(path_z, .{ .ACCMODE = .RDONLY });
    if (fd < 0) return error.OpenFailed;
    defer _ = libc.close(fd);
    const end = libc.lseek(fd, 0, libc.SEEK.END);
    if (end < 0 or libc.lseek(fd, 0, libc.SEEK.SET) < 0) return error.ReadFailed;
    const len: usize = @intCast(end);
    const buf = try gpa.alloc(u8, len + 1);
    errdefer gpa.free(buf);
    var filled: usize = 0;
    while (filled < len) {
        const n = libc.read(fd, buf[filled..].ptr, len - filled);
        if (n < 0) {
            if (libc.errno(n) == .INTR) continue;
            return error.ReadFailed;
        }
        if (n == 0) return error.UnexpectedEof;
        filled += @intCast(n);
    }
    buf[len] = 0;
    return buf[0..len :0];
}

fn rssKib() u64 {
    const fd = libc.open("/proc/self/status", .{ .ACCMODE = .RDONLY });
    if (fd < 0) return 0;
    defer _ = libc.close(fd);
    var buf: [4096]u8 = undefined;
    const n = libc.read(fd, &buf, buf.len);
    if (n <= 0) return 0;
    var it = std.mem.tokenizeScalar(u8, buf[0..@intCast(n)], '\n');
    while (it.next()) |line| {
        if (!std.mem.startsWith(u8, line, "VmHWM:")) continue;
        var t = std.mem.tokenizeAny(u8, line["VmHWM:".len..], " \t");
        const v = t.next() orelse return 0;
        return std.fmt.parseInt(u64, v, 10) catch 0;
    }
    return 0;
}

// ---- probe: one query from the command line ----

fn nowUs() u64 {
    var ts: std.c.timespec = undefined;
    _ = std.c.clock_gettime(.MONOTONIC, &ts);
    return @as(u64, @intCast(ts.sec)) *| 1_000_000 +| @as(u64, @intCast(ts.nsec)) / 1000;
}

const aliases = .{
    .{ "gd", lsp.Kind.definition },
    .{ "gD", lsp.Kind.declaration },
    .{ "gy", lsp.Kind.type_definition },
    .{ "gi", lsp.Kind.implementation },
    .{ "gr", lsp.Kind.references },
};

fn probeUsage() noreturn {
    std.debug.print(
        \\usage: pardes-lspbench probe <kind> <file> <line>:<col> [arg] [--reps N] [--warm S]
        \\  kind:   gd gD gy gi gr, or any lsp.Kind tag (hover, references,
        \\          document_symbols, workspace_symbols, incoming_calls, ...)
        \\  arg:    the new name for rename, the query for workspace_symbols
        \\  --warm  sleep S seconds after the first (cold) query — long enough
        \\          for an indexing server to reach ready, narrated on stderr
        \\
    , .{});
    std.process.exit(2);
}

/// The editor's message row, here: the client's reader threads narrate server
/// state through the same sink the shells register, printed to stderr so the
/// rows on stdout stay clean.
fn statusSink(ctx: ?*anyopaque, text: []const u8) void {
    _ = ctx;
    std.debug.print("# status: {s}\n", .{text});
}

extern "c" fn usleep(usec: c_uint) c_int;

/// `lspbench probe`: one language query against the real seam, from the
/// command line (the old lspprobe): `probe <kind> <file> <line>:<col> [arg]
/// [--reps N] [--warm S]`, rows on stdout, the timing on stderr.
fn probeCommand(init: std.process.Init) !void {
    const gpa = std.heap.page_allocator;
    const arena = init.arena.allocator();
    const io = std.Io.Threaded.global_single_threaded.io();

    // args[1] is `probe`: the rest are where lspprobe's were.
    const all = try init.minimal.args.toSlice(arena);
    const args = all[1..];
    if (args.len < 4) probeUsage();

    // comma-separated kinds share ONE server instance and its index — the
    // whole battery against a big workspace pays the cold cost once
    var kinds: [16]lsp.Kind = undefined;
    var nkinds: usize = 0;
    var kit = std.mem.tokenizeScalar(u8, args[1], ',');
    while (kit.next()) |word| {
        if (nkinds == kinds.len) probeUsage();
        kinds[nkinds] = blk: {
            inline for (aliases) |a| if (std.mem.eql(u8, word, a[0])) break :blk a[1];
            break :blk std.meta.stringToEnum(lsp.Kind, word) orelse probeUsage();
        };
        nkinds += 1;
    }
    if (nkinds == 0) probeUsage();

    var pz: [4096:0]u8 = undefined;
    const pz_s = std.fmt.bufPrintSentinel(&pz, "{s}", .{args[2]}, 0) catch probeUsage();
    var realbuf: [4096]u8 = undefined;
    const path = std.mem.span(realpath(pz_s.ptr, &realbuf) orelse {
        std.debug.print("cannot resolve {s}\n", .{args[2]});
        std.process.exit(1);
    });
    const source = std.Io.Dir.cwd().readFileAllocOptions(
        io,
        path,
        arena,
        .limited(64 << 20),
        .of(u8),
        0,
    ) catch {
        std.debug.print("cannot read {s}\n", .{path});
        std.process.exit(1);
    };

    const colon = std.mem.indexOfScalar(u8, args[3], ':') orelse probeUsage();
    const line = std.fmt.parseInt(u32, args[3][0..colon], 10) catch probeUsage();
    const col = std.fmt.parseInt(u32, args[3][colon + 1 ..], 10) catch probeUsage();
    if (line == 0 or col == 0) probeUsage();

    var arg: []const u8 = "";
    var reps: u32 = 1;
    var warm_s: u32 = 0;
    var i: usize = 4;
    while (i < args.len) : (i += 1) {
        if (std.mem.eql(u8, args[i], "--reps")) {
            i += 1;
            if (i >= args.len) probeUsage();
            reps = @max(1, std.fmt.parseInt(u32, args[i], 10) catch probeUsage());
        } else if (std.mem.eql(u8, args[i], "--warm")) {
            i += 1;
            if (i >= args.len) probeUsage();
            warm_s = std.fmt.parseInt(u32, args[i], 10) catch probeUsage();
        } else arg = args[i];
    }

    lsp.setStatusSink(null, statusSink);

    // 1-based line:col -> byte offset
    var off: usize = 0;
    var ln: u32 = 1;
    while (ln < line and off < source.len) : (off += 1) {
        if (source[off] == '\n') ln += 1;
    }
    off = @min(off + col - 1, source.len);

    const root = std.fs.path.dirname(path) orelse "/";
    const timings = try arena.alloc(u64, reps);
    var stdout_buf: [64 * 1024]u8 = undefined;
    var stdout = std.Io.File.stdout().writer(io, &stdout_buf);
    var first_query = true;

    for (kinds[0..nkinds]) |kind| {
        var rows: []const u8 = "";
        for (0..reps) |r| {
            // one warm pause for the whole battery, right after the very
            // first (cold, server-spawning) query — the status lines on
            // stderr narrate what the server does with the time
            var out: std.Io.Writer.Allocating = .init(gpa);
            defer out.deinit();
            var scratch: std.heap.ArenaAllocator = .init(gpa);
            defer scratch.deinit();
            const t0 = nowUs();
            try lsp.query(gpa, scratch.allocator(), .{
                .kind = kind,
                .path = path,
                .source = source,
                .offset = @intCast(off),
                .arg = arg,
                .root = root,
            }, &out.writer);
            timings[r] = nowUs() -| t0;
            rows = try arena.dupe(u8, out.written());
            if (first_query) {
                first_query = false;
                if (warm_s > 0) for (0..warm_s * 10) |_| {
                    _ = usleep(100_000);
                };
            }
        }

        try stdout.interface.print("## {t}\n", .{kind});
        try stdout.interface.writeAll(rows);
        try stdout.interface.flush();

        const n = std.mem.count(u8, rows, "\n");
        const cold = timings[0];
        std.mem.sort(u64, timings[0..reps], {}, std.sort.asc(u64));
        if (reps == 1)
            std.debug.print("# {t}: {d} row(s) in {d}us\n", .{ kind, n, cold })
        else
            std.debug.print("# {t}: {d} row(s); first {d}us, min {d}us, median {d}us ({d} reps)\n", .{
                kind, n, cold, timings[0], timings[reps / 2], reps,
            });
    }
}