diff options
| -rw-r--r-- | docs/lsp-evaluation.md | 20 | ||||
| -rw-r--r-- | docs/typ/builtins-gui.json | 1 | ||||
| -rw-r--r-- | docs/typ/builtins-tui.json | 1 | ||||
| -rw-r--r-- | docs/typ/guide.typ | 6 | ||||
| -rw-r--r-- | docs/typ/reference.typ | 22 | ||||
| -rw-r--r-- | docs/typ/setup.typ | 10 | ||||
| -rw-r--r-- | src/CHANGELOG.md | 16 | ||||
| -rw-r--r-- | src/Output.zig | 2 | ||||
| -rw-r--r-- | src/builtins.zig | 7 | ||||
| -rw-r--r-- | src/config.zig | 1 | ||||
| -rw-r--r-- | src/host_io.zig | 3 | ||||
| -rw-r--r-- | src/lsp/Scip.zig | 354 | ||||
| -rw-r--r-- | src/lsp/lsp.zig | 7 | ||||
| -rw-r--r-- | src/lsp/lsp_client.zig | 426 | ||||
| -rw-r--r-- | src/pardes.zig | 7 | ||||
| -rw-r--r-- | test/behaviour.zig | 1 | ||||
| -rw-r--r-- | test/behaviour/language_servers.zig | 294 | ||||
| -rw-r--r-- | test/process/builtins.zig | 4 | ||||
| -rw-r--r-- | test/scipfixture/Cargo.toml | 11 | ||||
| -rw-r--r-- | test/scipfixture/index.scip | bin | 0 -> 1745 bytes | |||
| -rw-r--r-- | test/scipfixture/lib.rs | 4 | ||||
| -rw-r--r-- | test/scipfixture/main.rs | 7 |
22 files changed, 1110 insertions, 94 deletions
diff --git a/docs/lsp-evaluation.md b/docs/lsp-evaluation.md index 0812ebf0..b98ea4c0 100644 --- a/docs/lsp-evaluation.md +++ b/docs/lsp-evaluation.md @@ -128,6 +128,26 @@ once, and its language words answer `language server for .x not found`. A (`nvqpknzm`), B (`stuurqqt`) and C's last state (the parent of `kvlpprms`) remain whole commits. +**2026-11-05: an index first, the server second.** The words ask the +nearest `index.scip` above the file (SCIP, scip-code.org: a protobuf file +an indexer writes at the project's root) before any server: +`src/lsp/Scip.zig` reads it, about 350 lines with its tests, keeping each +document's occurrences and each symbol's information in a fixed region +(an index of 32 MB at most, its tables 32 MB, two indexes at once). Def, +type definition, Refs, SelectRefs, `gi`, Subtypes, Supertypes, Symbols, +WsSymbols, Hover, Callers, Callees and the Callstack tree answer from it +where it knows the symbol: locally, with no handshake and no index to +wait for, the Callstack's callers each the function whose definition holds +the call. The server stays for what an index cannot say (Diagnostics, +Rename, CodeAction, format, completion, declaration), for a pane with +unsaved changes, and as the fallback for a file, a place or a symbol the +index lacks, the call hierarchy included (against the architecture's +§13, which dropped it: Callstack over clangd and rust-analyzer stays for +a project with no index). Measured on rust-analyzer's index of +clap_builder 4.6.7 (3.9 MB, 32,795 occurrences, 3.1 MB of tables), in a +Debug core: the first question reads and parses the index in 3.7 ms, a +`gd` after it takes 0.14 ms, `gr` 0.6 ms, a 7,463-row Callstack 62 ms. + ## Recommendation (as written before the decision) **C (in-process), with B as the answer to a question pardes has not asked yet.** diff --git a/docs/typ/builtins-gui.json b/docs/typ/builtins-gui.json index 9a8b2ced..e48dc0cd 100644 --- a/docs/typ/builtins-gui.json +++ b/docs/typ/builtins-gui.json @@ -53,6 +53,7 @@ {"name": "HoverGlow", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": "on, off", "doc": "A soft underline of light under the word a look-hover would open, on or off."}, {"name": "InactiveDim", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": null, "doc": "Fade the unfocused panes' text by a percentage, 0 through 100, 0 off."}, {"name": "Incl", "leader": null, "also": [], "arg": true, "scope": "root", "setting": false, "choices": null, "doc": "acme's Incl, for the session: `Incl dir ...` adds directories a look at a name found nowhere else is tried in (`stdio.h`, `<foo.h>`), bare lists them, `Incl -` empties the list."}, + {"name": "Index", "leader": null, "also": [], "arg": false, "scope": "pane", "setting": false, "choices": null, "doc": "Run the file's language indexer at its project root, writing the index.scip the language words ask first."}, {"name": "Joincol", "leader": "SPC c j", "also": ["topbar"], "arg": false, "scope": "root", "setting": false, "choices": null, "doc": "Fold the active pane's column into the one on its right, keeping its panes."}, {"name": "JumpScope", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": "file, all", "doc": "What Back and Forward keep to: file, the focused pane's file until it runs out that way, or all, every place in order."}, {"name": "JumpTrail", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": "on, off", "doc": "In a terminal, a jump of the cursor leaves a short trail that fades, on or off."}, diff --git a/docs/typ/builtins-tui.json b/docs/typ/builtins-tui.json index f5fe199c..93ae6a8f 100644 --- a/docs/typ/builtins-tui.json +++ b/docs/typ/builtins-tui.json @@ -45,6 +45,7 @@ {"name": "Hover", "leader": "SPC l k", "also": [], "arg": false, "scope": "pane", "setting": false, "choices": null, "doc": "Ask the language server about the symbol under the cursor."}, {"name": "InactiveDim", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": null, "doc": "Fade the unfocused panes' text by a percentage, 0 through 100, 0 off."}, {"name": "Incl", "leader": null, "also": [], "arg": true, "scope": "root", "setting": false, "choices": null, "doc": "acme's Incl, for the session: `Incl dir ...` adds directories a look at a name found nowhere else is tried in (`stdio.h`, `<foo.h>`), bare lists them, `Incl -` empties the list."}, + {"name": "Index", "leader": null, "also": [], "arg": false, "scope": "pane", "setting": false, "choices": null, "doc": "Run the file's language indexer at its project root, writing the index.scip the language words ask first."}, {"name": "Joincol", "leader": "SPC c j", "also": ["topbar"], "arg": false, "scope": "root", "setting": false, "choices": null, "doc": "Fold the active pane's column into the one on its right, keeping its panes."}, {"name": "JumpScope", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": "file, all", "doc": "What Back and Forward keep to: file, the focused pane's file until it runs out that way, or all, every place in order."}, {"name": "JumpTrail", "leader": null, "also": [], "arg": true, "scope": "root", "setting": true, "choices": "on, off", "doc": "In a terminal, a jump of the cursor leaves a short trail that fades, on or off."}, diff --git a/docs/typ/guide.typ b/docs/typ/guide.typ index 29535c39..7c5b4a8e 100644 --- a/docs/typ/guide.typ +++ b/docs/typ/guide.typ @@ -279,8 +279,10 @@ continue alongside every line of a multiline signature and through screen wrapping. Cross-directory Look paths are shortened when possible. #btn("B3") or #key("Enter") on a location opens and reveals that source pane. #word("LookNext") / #word("LookPrev") (#keys("SPC j n", "SPC j p")) -select locations without opening them. The view follows the language -server's static call hierarchy; recursive paths are marked `↻`, and a +select locations without opening them. The view follows the static call +hierarchy of the project's `index.scip` (each caller the function whose +definition holds the call), else the language server's; recursive paths +are marked `↻`, and a hierarchy deeper than the limit invites continuing from another function. Every caller is listed: the walk runs as long as the server keeps answering, says how many callers it has found on the message row once it takes more diff --git a/docs/typ/reference.typ b/docs/typ/reference.typ index d3210814..a7db9e2d 100644 --- a/docs/typ/reference.typ +++ b/docs/typ/reference.typ @@ -254,7 +254,16 @@ A line written to #file("exec") is #word("Exec"): - A line starting with `#` is a comment, as in a shell: it runs as nothing, silently, here and in every ctl. - The language server's words ask about a file pane's text at its cursor - (set it with #file("addr") and `dot=addr` first): #word("Hover") fills + (set it with #file("addr") and `dot=addr` first). Each asks the nearest + `index.scip` above the file first (a SCIP index an indexer wrote, read + again once it changes; #word("Index") runs the file's indexer at its + project's root in a command pane) and the file's language server only + when the index has nothing to say: none above the file, a file it does + not hold, nothing indexed at the cursor, a symbol it does not know, a + pane with unsaved changes, or a word only a server answers (declaration, + #word("Diagnostics"), #word("Rename"), #word("CodeAction"), format, + completion). An index older than the file still answers, and says so + once on the message row. #word("Hover") fills `+Hover`; #word("Def") and #word("Refs"), in the tag of a file a server speaks for, are `gd` and `gr`, and given a symbol (a 2-1 chord's selection) ask at its occurrence nearest the cursor; #word("Rename") `new` renames the symbol in the file and says how many @@ -275,10 +284,13 @@ A line written to #file("exec") is #word("Exec"): #word("Callers") and #word("Callees") keep their short lists and single-result jumps. #word("Diagnostics") and #word("Symbols") list the file's in `+Search`; - #word("Lspinfo") fills `+Lsp`, the pane its write answers, with which - server serves the file and its state (`not started yet` until its first - query starts it, `<server> failed recently; retry in Ns` while it backs - off); + #word("Lspinfo") fills `+Lsp`, the pane its write answers, with the + index the file asks (`scip: <path>, <size>, <documents>, <symbols>, + indexed <when>, <n> files newer`, or `scip: no index.scip above <dir>; + Index runs <indexer>`), which server serves it and its state (`not + started yet` until its first query starts it, `<server> failed recently; + retry in Ns` while it backs off), and what answered each query lately + (`scip`, or the server); #word("Lspwhy") narrates the last query step by step (in `+Lsp`). The write returns once the answer is in; a pane with no file is refused. - acme's words run as pardes's where it has one (`Put` is #word("Save"), diff --git a/docs/typ/setup.typ b/docs/typ/setup.typ index f49c9075..466b178a 100644 --- a/docs/typ/setup.typ +++ b/docs/typ/setup.typ @@ -117,6 +117,16 @@ pardes starts a language server for a file it knows, from `PATH`: #word("Lspinfo") (#key("SPC l i")) says what is running. +Before a server, a language word asks the nearest `index.scip` above the +file, at most seven directories up: a SCIP index (scip-code.org) of the +project, which answers its words at once, with no server running. +#word("Index") runs the file's indexer at the project's root in a command +pane, and the index is read again once the indexer is done: `rust-analyzer +scip .`, `scip-clang`, `scip-go`, `scip-typescript index`, `scip-python +index .` or `scip-zig` (`PARDES_SCIP_RS` and the like name another, as +`PARDES_LSP_RS` does a server). Nothing indexes behind your back: run it +again after big changes. An index of more than 32 MB is refused. + = Fonts and themes <fonts-and-themes> The terminal build uses your terminal's font. The SDL window carries diff --git a/src/CHANGELOG.md b/src/CHANGELOG.md index e91bb289..d90143c4 100644 --- a/src/CHANGELOG.md +++ b/src/CHANGELOG.md @@ -124,10 +124,18 @@ Signatures use syntax colors; only calls to the queried function get background emphasis, and tree guides continue through multiline signatures and wrapped rows. Cross-directory call locations use shorter relative paths. -- Lspinfo's `+Lsp` starts with its language servers: the `backend: - lsp-client` line, left from when Zig files had an in-process backend of - their own, is gone. A language word in a file no server speaks for names - the file (`Hover: no language server for notes.txt`). +- The language words ask an `index.scip` first (SCIP, scip-code.org): the + nearest above the file answers Def, Refs, `gi`, Subtypes, Supertypes, + Symbols, WsSymbols, Hover, Callers, Callees and Callstack where it knows + the symbol, at once and with no server; the language server answers + what it does not know, a pane's unsaved changes, and Diagnostics, + Rename, CodeAction, format and completion. `Index` runs the file's + indexer at the project's root (`rust-analyzer scip .` for Rust), and + Lspinfo starts with the index the file would read. +- Lspinfo's `+Lsp` no longer says `backend: lsp-client`, a line left from + when Zig files had an in-process backend of their own. A language word + in a file no server speaks for names the file (`Hover: no language + server for notes.txt`). - A terminal follows the shell at its prompt (the tty's foreground group), not the process pardes started: a shell that re-execs itself under a wrapper (fish's `exec 9ns --mntgen -- fish`) no longer leaves the pane diff --git a/src/Output.zig b/src/Output.zig index abdf2664..3f869154 100644 --- a/src/Output.zig +++ b/src/Output.zig @@ -265,7 +265,7 @@ pub fn traits(o: Origin) Traits { .hover => .{ .name = config.hover_buffer }, .callstack => .{ .name = "+Callstack", .steps = true, .locations = true }, // prose: an action list, a diff, a report about the backend - .code_action, .format, .status, .explain => .{ .name = config.lsp_buffer }, + .code_action, .format, .status, .explain, .index => .{ .name = config.lsp_buffer }, .rename => .{ .name = config.search_buffer, .steps = true, .locations = true }, .definition, .declaration, .type_definition, .implementation, .references => .{ .name = config.search_buffer, diff --git a/src/builtins.zig b/src/builtins.zig index c2be41a8..d203cd9e 100644 --- a/src/builtins.zig +++ b/src/builtins.zig @@ -1970,6 +1970,13 @@ pub const WsDiagnostics = struct { } }; +/// Run the file's language indexer at its project root, writing the index.scip the language words ask first. +pub const Index = struct { + pub fn run(c: Ctx) void { + c.p.lspRequest(c.id, .index, ""); + } +}; + /// Show outermost callers first, with signatures and call sites (gc). pub const Callstack = struct { pub fn run(c: Ctx) void { diff --git a/src/config.zig b/src/config.zig index 641ff463..0227f898 100644 --- a/src/config.zig +++ b/src/config.zig @@ -64,6 +64,7 @@ pub const leader_path = paths: { .Lspinfo = "li", .Lspwhy = "lw", .Callstack = "lb", + .Index = null, .Callers = "lc", .Callees = "lC", .Supertypes = "lt", diff --git a/src/host_io.zig b/src/host_io.zig index a04bf7ce..d13b7643 100644 --- a/src/host_io.zig +++ b/src/host_io.zig @@ -617,6 +617,7 @@ pub const Lsp = struct { /// One language query, owned by the worker that runs it. pub const Job = struct { callstack_depth: u8 = pardes.lsp.callstack_default_depth, + dirty: bool = false, id: u32, kind: pardes.lsp.Kind, offset: u32, @@ -660,6 +661,7 @@ pub const Lsp = struct { errdefer gpa.free(root); job.* = .{ .callstack_depth = core.settings.callstack_depth, + .dirty = pardes.tagline.dirty(pane), .id = req.id, .kind = req.kind, .offset = req.offset, @@ -746,6 +748,7 @@ pub const Lsp = struct { .root = job.root, .others = job.others, .callstack_depth = job.callstack_depth, + .dirty = job.dirty, .budget = &arena, }, &out.writer) catch { deliver(ctx, job.id, null); diff --git a/src/lsp/Scip.zig b/src/lsp/Scip.zig new file mode 100644 index 00000000..8deffe26 --- /dev/null +++ b/src/lsp/Scip.zig @@ -0,0 +1,354 @@ +//! A repository's code intelligence, read from its `index.scip` (SCIP, +//! scip-code.org; the schema is scip.proto in sourcegraph/scip): the +//! language words ask it first and the language server when it has +//! nothing to say (lsp_client.zig). An indexer writes it at the project's +//! root (the Index word runs one); the nearest above a file's directory +//! answers for the file. It is read on the first question that wants it, +//! on the query's worker, off the typing path, and again once it changes. +//! +//! Kept: each document's path, position encoding and occurrences (range, +//! symbol, roles, the enclosing node's range), the symbols interned with +//! their occurrences listed by symbol, and the bytes of each symbol's +//! information, read again when a word wants its documentation, signature, +//! kind or relationships. Skipped: the metadata (paths resolve against +//! the index's own directory), external symbols, syntax kinds, an +//! occurrence's diagnostics and a document's text. +const std = @import("std"); +const statPath = @import("../fs.zig").statPath; + +/// One loaded index's caps, measured on a real one: rust-analyzer 1.97's +/// index of clap_builder 4.6.7 (29 kloc) is 3.9 MB, with 55 documents, +/// 32,795 occurrences and 6,661 symbols, and its tables take 3.1 MB; these +/// hold eight times that. A bigger index is refused, in words. +pub const max_bytes = 32 << 20; +const tables_bytes = 32 << 20; +/// Indexes kept loaded at once, the one asked least recently replaced. +const kept = 2; + +pub const Range = struct { sl: u32 = 0, sc: u32 = 0, el: u32 = 0, ec: u32 = 0 }; +/// What a document's characters count (its `position_encoding`). +pub const Enc = enum { utf8, utf16, utf32 }; +/// An occurrence's role bit (scip.proto SymbolRole) for a definition. +pub const role_definition = 1; + +pub const Occurrence = struct { + range: Range, + /// The node a definition names, a function's signature and body, when + /// the indexer gave it; empty otherwise. + enclosing: Range, + symbol: u32, + doc: u32, + roles: u32, +}; + +pub const Document = struct { path: []const u8, enc: Enc, first: u32, len: u32 }; + +pub const Index = struct { + path_buf: [4096]u8 = undefined, + path_len: usize = 0, + mtime: i96 = 0, + size: u64 = 0, + used: u64 = 0, + /// A document newer than the index has been said once. + stale_said: bool = false, + documents: []Document = &.{}, + occurrences: []Occurrence = &.{}, + /// By symbol id: its string, its information's bytes (empty for one + /// defined in no document here), and its occurrences, + /// `order[starts[id]..starts[id + 1]]`. + symbols: std.ArrayList([]const u8) = .empty, + infos: std.ArrayList([]const u8) = .empty, + starts: []u32 = &.{}, + order: []u32 = &.{}, + ids: std.StringHashMapUnmanaged(u32) = .empty, + + pub fn path(ix: *const Index) []const u8 { + return ix.path_buf[0..ix.path_len]; + } + + pub fn root(ix: *const Index) []const u8 { + return std.fs.path.dirname(ix.path()) orelse "/"; + } + + /// The document of the file at `abs`, or null. + pub fn document(ix: *const Index, abs: []const u8) ?u32 { + const r = ix.root(); + if (abs.len <= r.len + 1 or !std.mem.startsWith(u8, abs, r) or abs[r.len] != '/') return null; + for (ix.documents, 0..) |d, i| if (std.mem.eql(u8, d.path, abs[r.len + 1 ..])) return @intCast(i); + return null; + } + + /// The innermost occurrence of `doc` at (line, ch), counted in the + /// document's units; one ending at the cursor counts. + pub fn at(ix: *const Index, doc: u32, line: u32, ch: u32) ?u32 { + const d = ix.documents[doc]; + var best: ?u32 = null; + for (d.first..d.first + d.len) |i| { + const r = ix.occurrences[i].range; + if (!holds(r, .{ .sl = line, .sc = ch, .el = line, .ec = ch })) continue; + if (best == null or holds(ix.occurrences[best.?].range, r)) best = @intCast(i); + } + return best; + } + + /// Whether the file at `abs` changed after the index was written. + pub fn stale(ix: *const Index, io: std.Io, abs: []const u8) bool { + const st = statPath(io, abs, .{}) catch return false; + return st.mtime.nanoseconds > ix.mtime; + } + + pub fn of(ix: *const Index, symbol: u32) []const u32 { + return ix.order[ix.starts[symbol]..ix.starts[symbol + 1]]; + } + + /// The definition whose node holds occurrence `o`, the innermost, of a + /// function-like symbol: where the call `o` is made from. + pub fn enclosingDefinition(ix: *const Index, o: u32) ?u32 { + const d = ix.documents[ix.occurrences[o].doc]; + var best: ?u32 = null; + for (d.first..d.first + d.len) |i| { + const def = ix.occurrences[i]; + if (i == o or def.roles & role_definition == 0 or !holds(def.enclosing, ix.occurrences[o].range)) continue; + if (!callable(ix.infos.items[def.symbol])) continue; + if (best == null or holds(ix.occurrences[best.?].enclosing, def.enclosing)) best = @intCast(i); + } + return best; + } +}; + +/// Whether `outer` holds `inner`, its ends included. +pub fn holds(outer: Range, inner: Range) bool { + const starts_before = outer.sl < inner.sl or (outer.sl == inner.sl and outer.sc <= inner.sc); + const ends_after = outer.el > inner.el or (outer.el == inner.el and outer.ec >= inner.ec); + return starts_before and ends_after and (outer.sl != outer.el or outer.sc != outer.ec); +} + +/// A symbol information's kind (5) is a function's or a method's. +pub fn callable(info: []const u8) bool { + const kind = (field(info, 5) orelse return false).int; + return switch (kind) { + 9, 17, 18, 25, 26, 45, 47, 66...72, 74, 76, 80 => true, + else => false, + }; +} + +// ------------------------------------------------------------------ the wire + +/// One field of a Protobuf message: its number, and its varint or its +/// length-delimited bytes (fixed 32- and 64-bit ones are skipped whole). +pub const Field = struct { num: u64, int: u64 = 0, bytes: []const u8 = "" }; + +pub const Reader = struct { + bytes: []const u8, + at: usize = 0, + + pub fn varint(r: *Reader) error{Malformed}!u64 { + var n: u64 = 0; + var shift: u7 = 0; + while (r.at < r.bytes.len and shift < 64) : (shift += 7) { + const b = r.bytes[r.at]; + r.at += 1; + n |= @as(u64, b & 0x7f) << @intCast(shift); + if (b < 0x80) return n; + } + return error.Malformed; + } + + pub fn next(r: *Reader) error{Malformed}!?Field { + if (r.at == r.bytes.len) return null; + const key = try r.varint(); + var f: Field = .{ .num = key >> 3 }; + switch (key & 7) { + 0 => f.int = try r.varint(), + 1 => r.at += 8, + 5 => r.at += 4, + 2 => { + const len = try r.varint(); + if (len > r.bytes.len - r.at) return error.Malformed; + f.bytes = r.bytes[r.at..][0..@intCast(len)]; + r.at += @intCast(len); + }, + else => return error.Malformed, + } + if (r.at > r.bytes.len) return error.Malformed; + return f; + } +}; + +/// The first field `num` of a message. +pub fn field(msg: []const u8, num: u64) ?Field { + var r: Reader = .{ .bytes = msg }; + while (r.next() catch return null) |f| if (f.num == num) return f; + return null; +} + +/// A range written as packed varints, `[line, start, end]` or `[line, +/// start, end line, end]`, or as a SingleLineRange or MultiLineRange +/// message (`typed`), its zeros left out. +fn rangeOf(bytes: []const u8, typed: enum { packed_ints, single, multi }) error{Malformed}!Range { + var v: [4]u32 = @splat(0); + var n: usize = 0; + var r: Reader = .{ .bytes = bytes }; + if (typed == .packed_ints) { + while (r.at < bytes.len and n < 4) : (n += 1) v[n] = @truncate(try r.varint()); + if (n == 3) return .{ .sl = v[0], .sc = v[1], .el = v[0], .ec = v[2] }; + if (n != 4) return error.Malformed; + } else while (try r.next()) |f| if (f.num >= 1 and f.num <= 4) { + v[f.num - 1] = @truncate(f.int); + }; + if (typed == .single) return .{ .sl = v[0], .sc = v[1], .el = v[0], .ec = v[2] }; + return .{ .sl = v[0], .sc = v[1], .el = v[2], .ec = v[3] }; +} + +// ------------------------------------------------------------------ loading + +var regions: [kept][max_bytes + tables_bytes]u8 align(std.heap.page_size_max) = @splat(@splat(0)); +var slots: [kept]Index = @splat(.{}); +var clock: u64 = 0; + +pub const Error = error{ TooBig, Malformed, Unreadable }; + +/// The index for a file in `dir`: the nearest `index.scip` there or up to +/// seven directories above, read again when it changed since; null with +/// none. +pub fn find(io: std.Io, dir: []const u8) Error!?*Index { + var buf: [4096]u8 = undefined; + var d = dir; + for (0..8) |_| { + const p = std.fmt.bufPrint(&buf, "{s}/index.scip", .{if (std.mem.eql(u8, d, "/")) "" else d}) catch return null; + if (statPath(io, p, .{})) |st| return try use(io, p, st) else |_| {} + d = std.fs.path.dirname(d) orelse return null; + } + return null; +} + +fn use(io: std.Io, p: []const u8, st: std.Io.File.Stat) Error!*Index { + clock += 1; + var pick: usize = 0; + for (&slots, 0..) |*ix, i| { + if (std.mem.eql(u8, ix.path(), p) and ix.mtime == st.mtime.nanoseconds and ix.size == st.size) { + ix.used = clock; + return ix; + } + if (ix.used < slots[pick].used) pick = i; + } + if (st.size > max_bytes) return error.TooBig; + const ix = &slots[pick]; + ix.* = .{ .mtime = st.mtime.nanoseconds, .size = st.size, .used = clock, .path_len = p.len }; + @memcpy(ix.path_buf[0..p.len], p); + errdefer ix.* = .{}; + const region = ®ions[pick]; + const file = std.Io.Dir.cwd().openFile(io, p, .{}) catch return error.Unreadable; + defer file.close(io); + const got = file.readPositionalAll(io, region[0..@intCast(st.size)], 0) catch return error.Unreadable; + var fba: std.heap.FixedBufferAllocator = .init(region[max_bytes..]); + parse(ix, region[0..got], fba.allocator()) catch |e| return if (e == error.OutOfMemory) error.TooBig else error.Malformed; + return ix; +} + +/// Two walks over the documents: one counts them and their occurrences, +/// the other fills the tables sized from the counts. +fn parse(ix: *Index, bytes: []const u8, a: std.mem.Allocator) !void { + var docs: usize = 0; + var occs: usize = 0; + var r: Reader = .{ .bytes = bytes }; + while (try r.next()) |f| if (f.num == 2) { + docs += 1; + var d: Reader = .{ .bytes = f.bytes }; + while (try d.next()) |g| occs += @intFromBool(g.num == 2); + }; + ix.documents = try a.alloc(Document, docs); + ix.occurrences = try a.alloc(Occurrence, occs); + occs = 0; + docs = 0; + r = .{ .bytes = bytes }; + while (try r.next()) |f| if (f.num == 2) { + const doc = &ix.documents[docs]; + doc.* = .{ .path = "", .enc = .utf8, .first = @intCast(occs), .len = 0 }; + var d: Reader = .{ .bytes = f.bytes }; + while (try d.next()) |g| switch (g.num) { + 1 => doc.path = g.bytes, + 2 => { + const o = &ix.occurrences[occs]; + o.* = .{ .range = .{}, .enclosing = .{}, .symbol = 0, .doc = @intCast(docs), .roles = 0 }; + var name: []const u8 = ""; + var e: Reader = .{ .bytes = g.bytes }; + while (try e.next()) |h| switch (h.num) { + 1 => o.range = try rangeOf(h.bytes, .packed_ints), + 2 => name = h.bytes, + 3 => o.roles = @truncate(h.int), + 7 => o.enclosing = try rangeOf(h.bytes, .packed_ints), + 8, 10 => (if (h.num == 8) &o.range else &o.enclosing).* = try rangeOf(h.bytes, .single), + 9, 11 => (if (h.num == 9) &o.range else &o.enclosing).* = try rangeOf(h.bytes, .multi), + else => {}, + }; + o.symbol = try intern(ix, a, name, docs); + occs += 1; + }, + 3 => if (field(g.bytes, 1)) |s| { + const id = try intern(ix, a, s.bytes, docs); + ix.infos.items[id] = g.bytes; + }, + 6 => doc.enc = switch (g.int) { + 2 => .utf16, + 3 => .utf32, + else => .utf8, + }, + else => {}, + }; + doc.len = @intCast(occs - doc.first); + docs += 1; + }; + // The occurrences listed by symbol: counted, the counts summed, filled. + ix.starts = try a.alloc(u32, ix.symbols.items.len + 1); + @memset(ix.starts, 0); + for (ix.occurrences) |o| ix.starts[o.symbol + 1] += 1; + for (1..ix.starts.len) |i| ix.starts[i] += ix.starts[i - 1]; + ix.order = try a.alloc(u32, ix.occurrences.len); + const filled = try a.alloc(u32, ix.symbols.items.len); + @memset(filled, 0); + for (ix.occurrences, 0..) |o, i| { + ix.order[ix.starts[o.symbol] + filled[o.symbol]] = @intCast(i); + filled[o.symbol] += 1; + } +} + +/// A symbol's id, a `local N` one's in its document alone (its key is the +/// document's number and the name), the first time it is met a new one. +fn intern(ix: *Index, a: std.mem.Allocator, name: []const u8, doc: usize) !u32 { + var buf: [64]u8 = undefined; + const local = std.mem.startsWith(u8, name, "local "); + const key = if (local) std.fmt.bufPrint(&buf, "{d} {s}", .{ doc, name }) catch return error.Malformed else name; + const got = try ix.ids.getOrPut(a, key); + if (!got.found_existing) { + if (local) got.key_ptr.* = try a.dupe(u8, key); + got.value_ptr.* = @intCast(ix.symbols.items.len); + try ix.symbols.append(a, name); + try ix.infos.append(a, ""); + } + return got.value_ptr.*; +} + +test "a varint and every wire type a reader skips" { + var r: Reader = .{ .bytes = &.{ 0xac, 0x02 } }; + try std.testing.expectEqual(@as(u64, 300), try r.varint()); + // field 1 varint 7, field 2 fixed64, field 3 bytes "hi", field 4 fixed32, field 5 varint 1 + const msg = [_]u8{ 0x08, 0x07, 0x11 } ++ @as([8]u8, @splat(9)) ++ [_]u8{ 0x1a, 0x02, 'h', 'i', 0x25 } ++ @as([4]u8, @splat(9)) ++ [_]u8{ 0x28, 0x01 }; + try std.testing.expectEqual(@as(u64, 7), field(&msg, 1).?.int); + try std.testing.expectEqualStrings("hi", field(&msg, 3).?.bytes); + try std.testing.expectEqual(@as(u64, 1), field(&msg, 5).?.int); + try std.testing.expect(field(&msg, 6) == null); + // cut short, an unknown wire type, a length past the end + for ([_][]const u8{ &.{0x80}, &.{ 0x0b, 0x00 }, &.{ 0x1a, 0x05, 'h' } }) |bad| { + var b: Reader = .{ .bytes = bad }; + try std.testing.expectError(error.Malformed, b.next()); + } +} + +test "ranges in the packed form and the typed ones, zeros left out" { + try std.testing.expectEqual(Range{ .sl = 3, .sc = 4, .el = 3, .ec = 7 }, try rangeOf(&.{ 3, 4, 7 }, .packed_ints)); + try std.testing.expectEqual(Range{ .sl = 1, .sc = 0, .el = 2, .ec = 5 }, try rangeOf(&.{ 1, 0, 2, 5 }, .packed_ints)); + try std.testing.expectEqual(Range{ .sl = 0, .sc = 0, .el = 0, .ec = 3 }, try rangeOf(&.{ 0x18, 3 }, .single)); + try std.testing.expectEqual(Range{ .sl = 2, .sc = 0, .el = 4, .ec = 1 }, try rangeOf(&.{ 0x08, 2, 0x18, 4, 0x20, 1 }, .multi)); + try std.testing.expectError(error.Malformed, rangeOf(&.{ 1, 2 }, .packed_ints)); +} diff --git a/src/lsp/lsp.zig b/src/lsp/lsp.zig index 038e4230..758d3042 100644 --- a/src/lsp/lsp.zig +++ b/src/lsp/lsp.zig @@ -23,6 +23,8 @@ pub const Kind = enum { subtypes, status, explain, + /// The Index word: the command line that makes the file's `index.scip`. + index, }; pub const callstack_match = "@callstack-match "; @@ -32,6 +34,9 @@ pub const callstack_max_depth: u8 = 64; pub const Req = struct { callstack_depth: u8 = callstack_default_depth, kind: Kind, + /// The pane holds changes not saved: what the index says of the file + /// on disk may be wrong of them, so the server answers (Scip.zig). + dirty: bool = false, path: []const u8, source: [:0]const u8, offset: u32, @@ -67,7 +72,7 @@ pub fn explaining(req: Req) Req { var r = req; const sp = std.mem.indexOfScalar(u8, req.arg, ' ') orelse req.arg.len; const named = std.meta.stringToEnum(Kind, req.arg[0..sp]) orelse .definition; - r.explains = if (named == .explain or named == .status) .definition else named; + r.explains = if (named == .explain or named == .status or named == .index) .definition else named; r.arg = if (sp < req.arg.len) req.arg[sp + 1 ..] else ""; return r; } diff --git a/src/lsp/lsp_client.zig b/src/lsp/lsp_client.zig index 40c287b0..928d5475 100644 --- a/src/lsp/lsp_client.zig +++ b/src/lsp/lsp_client.zig @@ -50,6 +50,7 @@ const std = @import("std"); const builtin = @import("builtin"); const libc = std.c; const lsp = @import("lsp.zig"); +const Scip = @import("Scip.zig"); extern "c" fn execv(path: [*:0]const u8, argv: [*:null]const ?[*:0]const u8) c_int; extern "c" fn usleep(usec: c_uint) c_int; @@ -81,6 +82,17 @@ const Spec = struct { /// empty ("") to disable the spec entirely. How the snapshot harness pins /// a deterministic mock server, and how a user points at a custom build. env: [:0]const u8, + /// The command that writes the language's `index.scip` at the project + /// root (Scip.zig), which the Index word runs there; `PARDES_SCIP_*` + /// (the env's language) names another. + indexer: []const u8, +}; + +/// Each spec's indexer override, `PARDES_SCIP_RS` beside `PARDES_LSP_RS`. +const indexer_envs = blk: { + var names: [specs.len][:0]const u8 = undefined; + for (specs, &names) |spec, *name| name.* = "PARDES_SCIP_" ++ spec.env["PARDES_LSP_".len..]; + break :blk names; }; pub const specs = [_]Spec{ @@ -91,6 +103,7 @@ pub const specs = [_]Spec{ .exts = &.{".zig"}, .markers = &.{ "build.zig", "build.zig.zon" }, .env = "PARDES_LSP_ZIG", + .indexer = "scip-zig", }, .{ .name = "rust-analyzer", @@ -99,6 +112,7 @@ pub const specs = [_]Spec{ .exts = &.{".rs"}, .markers = &.{ "Cargo.toml", "rust-project.json" }, .env = "PARDES_LSP_RS", + .indexer = "rust-analyzer scip .", }, .{ .name = "clangd", @@ -107,6 +121,7 @@ pub const specs = [_]Spec{ .exts = &.{ ".c", ".h", ".cc", ".cpp", ".hpp", ".cxx", ".hxx" }, .markers = &.{ "compile_commands.json", "compile_flags.txt", ".clangd" }, .env = "PARDES_LSP_C", + .indexer = "scip-clang", }, .{ .name = "gopls", @@ -115,6 +130,7 @@ pub const specs = [_]Spec{ .exts = &.{".go"}, .markers = &.{ "go.mod", "go.work" }, .env = "PARDES_LSP_GO", + .indexer = "scip-go", }, .{ .name = "typescript-language-server", @@ -124,6 +140,7 @@ pub const specs = [_]Spec{ .exts = &.{ ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs" }, .markers = &.{ "tsconfig.json", "jsconfig.json", "package.json" }, .env = "PARDES_LSP_TS", + .indexer = "scip-typescript index", }, .{ .name = "pyright", @@ -133,6 +150,7 @@ pub const specs = [_]Spec{ .exts = &.{".py"}, .markers = &.{ "pyproject.toml", "setup.py", "requirements.txt" }, .env = "PARDES_LSP_PY", + .indexer = "scip-python index .", }, }; @@ -200,7 +218,7 @@ const sa = std.heap.page_allocator; /// Which units `character` counts in — see the header note on believing the /// server. -const Enc = enum { utf8, utf16 }; +const Enc = Scip.Enc; const Doc = struct { uri: []u8, @@ -391,6 +409,11 @@ fn record(si: usize, req: lsp.Req, us: u64, rows: usize, err: []const u8) void { log_total += 1; } +/// What answered a logged query: a server, or `scip`, the index. +fn sourceName(spec: u8) []const u8 { + return if (spec < specs.len) specs[spec].name else "scip"; +} + fn hideTime() bool { return @import("../Messages.zig").Message.noTime(); } @@ -414,9 +437,27 @@ const Trace = struct { pub fn query(io: std.Io, arena: std.mem.Allocator, asked: lsp.Req, out: *std.Io.Writer) !void { const req = lsp.explaining(asked); - if (req.kind == .status) return status(io, out); + if (req.kind == .status) return status(io, req.path, out); + if (req.kind == .index) return indexCommand(req, out); var tr: Trace = .{ .on = req.kind == .explain }; + // Raw: `alloc` fills a safe build's new memory, all 51 MB of it, per query. + const scratch_buf = (arena.rawAlloc(max_rows * 512, .of(u8), @returnAddress()) orelse return error.OutOfMemory)[0 .. max_rows * 512]; + var scratch: std.Io.Writer = .fixed(scratch_buf); + + // The index first: what it answers wants no server. + const t0 = nowUs(); + if (fromIndex(io, arena, req, if (req.kind == .explain) req.explains else req.kind, &scratch, &tr) catch |e| blk: { + tr.note("index: {s}", .{@errorName(e)}); + break :blk false; + }) { + const us = nowUs() -| t0; + const rows = std.mem.count(u8, scratch.buffered(), "\n"); + record(specs.len, req, us, rows, ""); + if (req.kind == .explain) return traceOut(&tr, req, out, rows, us); + return out.writeAll(scratch.buffered()); + } + scratch.end = 0; const si = specFor(req.path) orelse { tr.note("STOP: no language server spec matches {s}", .{ if (req.path.len == 0) "a pane with no file" else req.path, @@ -426,11 +467,7 @@ pub fn query(io: std.Io, arena: std.mem.Allocator, asked: lsp.Req, out: *std.Io. }; tr.note("file {s} -> {s} (languageId {s})", .{ std.fs.path.basename(req.path), specs[si].name, specs[si].lang }); - // Raw: `alloc` fills a safe build's new memory, all 51 MB of it, per query. - const scratch_buf = (arena.rawAlloc(max_rows * 512, .of(u8), @returnAddress()) orelse return error.OutOfMemory)[0 .. max_rows * 512]; - var scratch: std.Io.Writer = .fixed(scratch_buf); - - const t0 = nowUs(); + const t1 = nowUs(); var err_name: []const u8 = ""; var failure: ?anyerror = null; answer(io, arena, si, req, &scratch, &tr) catch |e| { @@ -438,7 +475,7 @@ pub fn query(io: std.Io, arena: std.mem.Allocator, asked: lsp.Req, out: *std.Io. err_name = @errorName(e); tr.note("ERROR: {s}", .{err_name}); }; - const us = nowUs() -| t0; + const us = nowUs() -| t1; const rows = std.mem.count(u8, scratch.buffered(), "\n"); record(si, req, us, rows, err_name); @@ -664,7 +701,7 @@ fn run(c: *Conn, io: std.Io, si: usize, arena: std.mem.Allocator, req: lsp.Req, .outgoing_calls => try hierarchy(c, io, si, arena, &cx, uri.items, pos, deadline, .outgoing, tr), .supertypes => try hierarchy(c, io, si, arena, &cx, uri.items, pos, deadline, .supers, tr), .subtypes => try hierarchy(c, io, si, arena, &cx, uri.items, pos, deadline, .subs, tr), - .status, .explain => unreachable, + .status, .explain, .index => unreachable, } } @@ -890,8 +927,8 @@ fn hierarchyResult(c: *Conn, io: std.Io, arena: std.mem.Allocator, item: std.jso } const CallTree = struct { - /// What a node keeps of the server's item: its own copies, so the - /// response it came in is freed once its subtree is walked. + /// What a node keeps of its caller: its own copies, so the response it + /// came in is freed once its subtree is walked. const Call = struct { uri: []const u8, name: []const u8, at: Range }; const Site = struct { range: Range, focus: bool }; const Node = struct { @@ -902,7 +939,7 @@ const CallTree = struct { }; nodes: std.ArrayList(Node) = .empty, roots: std.ArrayList(usize) = .empty, - path: [lsp.callstack_max_depth]std.json.Value = undefined, + path: [lsp.callstack_max_depth]Hop = undefined, depth_limit: usize = lsp.callstack_default_depth, limited: bool = false, out_of_time: bool = false, @@ -931,21 +968,20 @@ const CallTree = struct { /// Incoming paths arrive nearest-first. Insert them outermost-first, /// sharing common caller prefixes and retaining all their call sites. - fn addPath(tree: *CallTree, arena: std.mem.Allocator, path: []const std.json.Value, cycle: bool) Err!void { + fn addPath(tree: *CallTree, arena: std.mem.Allocator, path: []const Hop, cycle: bool) Err!void { var parent: ?usize = null; var at = path.len; while (at > 0) { at -= 1; - const entry = path[at]; - const item = get(entry, "from").?; + const hop = path[at]; const children = if (parent) |id| tree.nodes.items[id].children.items else tree.roots.items; const found = for (children) |id| { - if (sameCall(tree.nodes.items[id].call, item)) break id; + if (sameCall(tree.nodes.items[id].call, hop.call)) break id; } else null; const depth = path.len - 1 - at; const id = found orelse blk: { const id = tree.nodes.items.len; - try tree.nodes.append(arena, .{ .call = try callOf(arena, item) }); + try tree.nodes.append(arena, .{ .call = .{ .uri = try arena.dupe(u8, hop.call.uri), .name = try arena.dupe(u8, hop.call.name), .at = hop.call.at } }); tree.rows += 1; tree.bytes += rowBytes(depth, tree.nodes.items[id].call); if (parent) |pid| { @@ -955,8 +991,7 @@ const CallTree = struct { }; const node = &tree.nodes.items[id]; node.cycle = node.cycle or (cycle and at == 0); - for (items(get(entry, "fromRanges"))) |range| { - const parsed = rangeOf(range) orelse continue; + for (hop.sites.items) |parsed| { for (node.ranges.items) |*other| { if (std.meta.eql(parsed, other.range)) { other.focus = other.focus or at == 0; @@ -973,37 +1008,58 @@ const CallTree = struct { } }; -fn callOf(arena: std.mem.Allocator, item: std.json.Value) Err!CallTree.Call { - return .{ - .uri = try arena.dupe(u8, str(get(item, "uri")).?), - .name = try arena.dupe(u8, str(get(item, "name")) orelse "(unnamed)"), - .at = rangeOf(get(item, "selectionRange") orelse get(item, "range")).?, - }; +/// A caller one level up a Callstack: the function, the calls it makes +/// there, and what its own callers are asked by, the server's item or the +/// index's symbol (and document). +const Hop = struct { call: CallTree.Call, sites: std.ArrayList(Range) = .empty, item: ?std.json.Value = null, symbol: u32 = 0, doc: u32 = 0 }; + +/// Where a Callstack's callers come from: the index, else the server. +const Calls = union(enum) { + index: *Scip.Index, + server: struct { c: *Conn, io: std.Io, parse: ?*std.heap.FixedBufferAllocator }, +}; + +fn sameCall(a: CallTree.Call, b: CallTree.Call) bool { + return std.mem.eql(u8, a.uri, b.uri) and std.meta.eql(a.at, b.at); } -fn sameCall(a: CallTree.Call, b: std.json.Value) bool { - const bu = str(get(b, "uri")) orelse return false; - const br = rangeOf(get(b, "selectionRange") orelse get(b, "range")) orelse return false; - return std.mem.eql(u8, a.uri, bu) and std.meta.eql(a.at, br); +/// The server's incoming calls as hops, their strings the response's. +fn serverHops(a: std.mem.Allocator, incoming: ?std.json.Value) Err![]Hop { + var hops: std.ArrayList(Hop) = .empty; + for (items(incoming)) |entry| { + const from = get(entry, "from") orelse continue; + const uri = str(get(from, "uri")) orelse continue; + const at = rangeOf(get(from, "selectionRange") orelse get(from, "range")) orelse continue; + if (!std.mem.startsWith(u8, uri, "file:")) continue; + var hop: Hop = .{ .call = .{ .uri = uri, .name = str(get(from, "name")) orelse "(unnamed)", .at = at }, .item = from }; + for (items(get(entry, "fromRanges"))) |r| if (rangeOf(r)) |site| try hop.sites.append(a, site); + try hops.append(a, hop); + } + return hops.items; } -fn sameItem(a: std.json.Value, b: std.json.Value) bool { - const au = str(get(a, "uri")) orelse return false; - const ar = rangeOf(get(a, "selectionRange") orelse get(a, "range")) orelse return false; - return sameCall(.{ .uri = au, .name = "", .at = ar }, b); +/// The index's callers of `symbol` as hops: the definition enclosing each +/// of its references, the references it holds its sites. +fn indexHops(a: std.mem.Allocator, ix: *Scip.Index, symbol: u32) Err![]Hop { + var hops: std.ArrayList(Hop) = .empty; + for (ix.of(symbol)) |o| { + if (ix.occurrences[o].roles & Scip.role_definition != 0) continue; + const def = ix.occurrences[ix.enclosingDefinition(o) orelse continue]; + const hop = for (hops.items) |*h| { + if (h.symbol == def.symbol) break h; + } else blk: { + try hops.append(a, .{ .call = .{ .uri = try indexUri(a, ix, def.doc), .name = displayName(ix, def.symbol), .at = def.range }, .symbol = def.symbol, .doc = def.doc }); + break :blk &hops.items[hops.items.len - 1]; + }; + try hop.sites.append(a, ix.occurrences[o].range); + } + return hops.items; } fn callPrefix(out: *std.Io.Writer, tree: *const CallTree, depth: usize) Err!void { for (tree.siblings[0..depth]) |more| try out.writeAll(if (more) "│ " else " "); } -fn validCaller(entry: std.json.Value) bool { - const from = get(entry, "from") orelse return false; - const uri = str(get(from, "uri")) orelse return false; - return std.mem.startsWith(u8, uri, "file:") and - rangeOf(get(from, "selectionRange") orelse get(from, "range")) != null; -} - fn callstack(c: *Conn, io: std.Io, si: usize, arena: std.mem.Allocator, cx: *Cx, uri: []const u8, pos: Pos, deadline: i64, tr: *Trace, max_depth: u8, budget: ?*const std.heap.FixedBufferAllocator) Err!void { if (!c.caps.call_hier) return lsp.none(cx.out, "{s} does not answer call hierarchy (no callHierarchyProvider)", .{specs[si].name}); const b = try atPos(arena, uri, pos); @@ -1021,46 +1077,52 @@ fn callstack(c: *Conn, io: std.Io, si: usize, arena: std.mem.Allocator, cx: *Cx, &.{}); const parse = if (budget != null) parse_fba.allocator() else arena; for (items(prepared)) |item| { - var tree: CallTree = .{ - .depth_limit = std.math.clamp(max_depth, 1, lsp.callstack_max_depth), - .si = si, - .started_ms = nowMs(), - .budget = budget, - // The walk stops while the answer's writer has room for the rest. - .byte_cap = if (cx.fixed_out) (cx.out.buffer.len - cx.out.end) -| (1 << 20) else std.math.maxInt(usize), - }; + const focus: CallTree.Call = .{ .uri = str(get(item, "uri")) orelse continue, .name = "", .at = rangeOf(get(item, "selectionRange") orelse get(item, "range")) orelse continue }; const incoming = try hierarchyResult(c, io, parse, item, "callHierarchy/incomingCalls", deadline); - const count = try collectCallers(c, io, arena, if (budget != null) &parse_fba else null, item, incoming, &tree, 0); - if (tree.narrated) post(si, .always, "Callstack: {d} callers", .{tree.collected}); - if (count == 0) try cx.out.writeAll("No callers reported\n"); - try renderCallers(cx, c.caps.enc, &tree, tree.roots.items, 0); - if (tree.out_of_time) { - try cx.out.writeAll("… the language server ran out of time; continue from a function above with Callstack\n"); - } else if (tree.out_of_memory) { - try cx.out.writeAll("… stopped at the query's memory limit; continue from a function above with Callstack\n"); - } else if (tree.limited) try cx.out.writeAll("… continue from a function above with Callstack\n"); - tr.note("callstack: {d} callers", .{tree.rendered}); + const src: Calls = .{ .server = .{ .c = c, .io = io, .parse = if (budget != null) &parse_fba else null } }; + try callTree(src, si, cx, c.caps.enc, focus, try serverHops(parse, incoming), tr, max_depth, budget); } } +/// A Callstack from the first callers of the function asked at: walked to +/// the depth setting, rendered, and why it stopped said. +fn callTree(src: Calls, si: usize, cx: *Cx, enc: Enc, focus: CallTree.Call, first: []const Hop, tr: *Trace, max_depth: u8, budget: ?*const std.heap.FixedBufferAllocator) Err!void { + var tree: CallTree = .{ + .depth_limit = std.math.clamp(max_depth, 1, lsp.callstack_max_depth), + .si = si, + .started_ms = nowMs(), + .budget = budget, + // The walk stops while the answer's writer has room for the rest. + .byte_cap = if (cx.fixed_out) (cx.out.buffer.len - cx.out.end) -| (1 << 20) else std.math.maxInt(usize), + }; + const count = try collectCallers(src, cx.arena, focus, first, &tree, 0); + if (tree.narrated) post(si, .always, "Callstack: {d} callers", .{tree.collected}); + if (count == 0) try cx.out.writeAll("No callers reported\n"); + try renderCallers(cx, enc, &tree, tree.roots.items, 0); + if (tree.out_of_time) { + try cx.out.writeAll("… the language server ran out of time; continue from a function above with Callstack\n"); + } else if (tree.out_of_memory) { + try cx.out.writeAll("… stopped at the query's memory limit; continue from a function above with Callstack\n"); + } else if (tree.limited) try cx.out.writeAll("… continue from a function above with Callstack\n"); + tr.note("callstack: {d} callers", .{tree.rendered}); +} + /// No caller count and no overall deadline: the walk narrates itself on the /// message row after a second, and it stops when a request goes unanswered /// or the answer is full (what was collected is shown, its last row saying /// so), or when the shell cancels it for a newer query. -fn collectCallers(c: *Conn, io: std.Io, arena: std.mem.Allocator, parse: ?*std.heap.FixedBufferAllocator, focus: std.json.Value, incoming: ?std.json.Value, tree: *CallTree, depth: usize) Err!usize { +fn collectCallers(src: Calls, arena: std.mem.Allocator, focus: CallTree.Call, hops: []const Hop, tree: *CallTree, depth: usize) Err!usize { var count: usize = 0; - for (items(incoming)) |entry| { + for (hops) |hop| { if (tree.budget) |b| if (b.buffer.len - b.end_index < callstack_reserve) { tree.out_of_memory = true; }; if (tree.rows >= tree.row_cap or tree.bytes >= tree.byte_cap) tree.out_of_memory = true; if (tree.out_of_time or tree.out_of_memory) break; - if (!validCaller(entry)) continue; - const from = get(entry, "from").?; - const cycle = sameItem(focus, from) or for (tree.path[0..depth]) |ancestor| { - if (sameItem(get(ancestor, "from").?, from)) break true; + const cycle = sameCall(focus, hop.call) or for (tree.path[0..depth]) |ancestor| { + if (sameCall(ancestor.call, hop.call)) break true; } else false; - tree.path[depth] = entry; + tree.path[depth] = hop; count += 1; tree.collected += 1; if (nowMs() - tree.started_ms >= 1000) { @@ -1071,12 +1133,16 @@ fn collectCallers(c: *Conn, io: std.Io, arena: std.mem.Allocator, parse: ?*std.h if (!cycle) { if (depth + 1 == tree.depth_limit) { tree.limited = true; + } else if (src == .index) { + children = try collectCallers(src, arena, focus, try indexHops(arena, src.index, hop.symbol), tree, depth + 1); } else { + const parse = src.server.parse; const mark = if (parse) |fixed| fixed.end_index else 0; defer if (parse) |fixed| { fixed.end_index = mark; }; - const ancestors = hierarchyResult(c, io, if (parse) |fixed| fixed.allocator() else arena, from, "callHierarchy/incomingCalls", nowMs() + req_ms) catch |e| switch (e) { + const a = if (parse) |fixed| fixed.allocator() else arena; + const ancestors = hierarchyResult(src.server.c, src.server.io, a, hop.item.?, "callHierarchy/incomingCalls", nowMs() + req_ms) catch |e| switch (e) { error.Timeout => blk: { tree.out_of_time = true; break :blk null; @@ -1089,7 +1155,7 @@ fn collectCallers(c: *Conn, io: std.Io, arena: std.mem.Allocator, parse: ?*std.h }, else => return e, }; - if (!tree.out_of_time and !tree.out_of_memory) children = try collectCallers(c, io, arena, parse, focus, ancestors, tree, depth + 1); + if (!tree.out_of_time and !tree.out_of_memory) children = try collectCallers(src, arena, focus, try serverHops(a, ancestors), tree, depth + 1); } } if (children == 0) try tree.addPath(arena, tree.path[0 .. depth + 1], cycle); @@ -1230,6 +1296,184 @@ fn hierText(arena: std.mem.Allocator, item: std.json.Value) []const u8 { return std.fmt.allocPrint(arena, "{s} {s}", .{ name, flat(arena, detail) }) catch name; } +// ---------------------------------------------------------------- the index + +/// The index's answer (Scip.zig), in the rows the server's would make: true +/// when it had one, false to ask the server: no index above the file, a +/// file it does not hold, unsaved changes it has not seen, nothing at the +/// cursor, a symbol it does not know, or a word only a server answers. +fn fromIndex(io: std.Io, arena: std.mem.Allocator, req: lsp.Req, kind: lsp.Kind, out: *std.Io.Writer, tr: *Trace) Err!bool { + switch (kind) { + .definition, .type_definition, .implementation, .references, .select_refs, .hover, .document_symbols, .workspace_symbols, .incoming_calls, .outgoing_calls, .callstack, .supertypes, .subtypes => {}, + else => return false, + } + if (req.dirty) { + tr.note("unsaved changes the index has not seen: the server", .{}); + return false; + } + const dir = std.fs.path.dirname(req.path) orelse return false; + const ix = (Scip.find(io, dir) catch |e| { + tr.note("index.scip above {s}: {s}", .{ dir, @errorName(e) }); + return false; + }) orelse { + tr.note("no index.scip above {s}", .{dir}); + return false; + }; + const doc = ix.document(req.path) orelse { + tr.note("{s} holds no {s}", .{ ix.path(), std.fs.path.basename(req.path) }); + return false; + }; + const si = specFor(req.path) orelse 0; + if (ix.stale(io, req.path)) { + tr.note("stale: {s} is newer than the index", .{std.fs.path.basename(req.path)}); + if (!ix.stale_said) post(si, .always, "{s} is older than {s}: Index makes it again", .{ ix.path(), std.fs.path.basename(req.path) }); + ix.stale_said = true; + } + const enc = ix.documents[doc].enc; + var cx: Cx = .{ .arena = arena, .base = req.root, .cur_path = req.path, .cur_src = req.source, .out = out, .fixed_out = true, .relative_parents = kind == .callstack }; + for (req.others) |d| try cx.files.append(arena, .{ .path = d.path, .text = d.source }); + if (kind == .document_symbols or kind == .workspace_symbols) { + for (ix.occurrences, 0..) |o, i| { + if (kind == .document_symbols and o.doc != doc) continue; + // A name, never a local or a whole file's module. + if (o.roles & Scip.role_definition == 0 or o.range.sl != o.range.el or std.mem.startsWith(u8, ix.symbols.items[o.symbol], "local ")) continue; + const name = displayName(ix, o.symbol); + if (kind == .workspace_symbols and std.ascii.findIgnoreCase(name, req.arg) == null) continue; + try emitOcc(&cx, ix, @intCast(i), name); + } + tr.note("<- {d} row(s) from the index", .{cx.rows}); + return cx.rows > 0; + } + const pos = posOf(req.source, req.offset, enc); + const at = ix.at(doc, pos.line, pos.ch) orelse { + tr.note("no occurrence at {d}:{d} in {s}", .{ pos.line + 1, pos.ch + 1, ix.path() }); + return false; + }; + const sym = ix.occurrences[at].symbol; + tr.note("{s}: occurrence '{s}' at {d}:{d}", .{ ix.path(), ix.symbols.items[sym], pos.line + 1, pos.ch + 1 }); + switch (kind) { + .definition => try emitDefinitions(&cx, ix, sym, false), + .type_definition => for (try related(arena, ix, sym, 4)) |to| try emitDefinitions(&cx, ix, to, false), + .supertypes => for (try related(arena, ix, sym, 3)) |to| try emitDefinitions(&cx, ix, to, true), + .implementation, .subtypes => for (ix.infos.items, 0..) |_, from| { + if (std.mem.indexOfScalar(u32, try related(arena, ix, @intCast(from), 3), sym) != null) + try emitDefinitions(&cx, ix, @intCast(from), kind == .subtypes); + }, + .references, .select_refs => { + var also = try related(arena, ix, sym, 2); + also = try std.mem.concat(arena, u32, &.{ &.{sym}, also }); + for (also) |of| for (ix.of(of)) |o| if (kind == .references or ix.occurrences[o].doc == doc) try emitOcc(&cx, ix, o, ""); + }, + .hover => { + const info = ix.infos.items[sym]; + var said = false; + if (Scip.field(info, 7)) |sig| if (Scip.field(sig.bytes, 5)) |text| { + try out.writeAll(std.mem.trim(u8, text.bytes, " \t\r\n")); + said = true; + }; + var r: Scip.Reader = .{ .bytes = info }; + while (r.next() catch null) |f| if (f.num == 3) { + if (said) try out.writeAll("\n\n"); + try out.writeAll(std.mem.trim(u8, f.bytes, " \t\r\n")); + said = true; + }; + if (said) try out.writeByte('\n') else tr.note("no documentation for it in the index", .{}); + return said; + }, + .incoming_calls => for (try indexHops(arena, ix, sym)) |hop| for (hop.sites.items) |site| + try emitPath(&cx, ix.documents[hop.doc].enc, try indexPath(arena, ix, hop.doc), site, hop.call.name), + .outgoing_calls => for (ix.of(sym)) |d| { + // The calls inside its definition, each function once, at its + // own definition. + const def = ix.occurrences[d]; + if (def.roles & Scip.role_definition == 0) continue; + var seen: std.ArrayList(u32) = .empty; + const in = ix.documents[def.doc]; + for (ix.occurrences[in.first..][0..in.len]) |o| { + if (o.roles & Scip.role_definition != 0 or !Scip.holds(def.enclosing, o.range) or !Scip.callable(ix.infos.items[o.symbol])) continue; + if (std.mem.indexOfScalar(u32, seen.items, o.symbol) != null) continue; + try seen.append(arena, o.symbol); + try emitDefinitions(&cx, ix, o.symbol, true); + } + break; + }, + .callstack => { + const def = for (ix.of(sym)) |o| { + if (ix.occurrences[o].roles & Scip.role_definition != 0) break ix.occurrences[o]; + } else { + tr.note("no definition of it in the index", .{}); + return false; + }; + // The rows come from the tree alone: the core reads the sources. + cx.no_text = true; + const focus: CallTree.Call = .{ .uri = try indexUri(arena, ix, def.doc), .name = "", .at = def.range }; + try callTree(.{ .index = ix }, si, &cx, enc, focus, try indexHops(arena, ix, sym), tr, req.callstack_depth, req.budget); + return true; + }, + else => unreachable, + } + tr.note("<- {d} row(s) from the index", .{cx.rows}); + return cx.rows > 0; +} + +/// A symbol's definitions, as rows: named by the symbol, as a type +/// hierarchy's are, or by their source lines. +fn emitDefinitions(cx: *Cx, ix: *Scip.Index, symbol: u32, named: bool) Err!void { + for (ix.of(symbol)) |o| if (ix.occurrences[o].roles & Scip.role_definition != 0) { + const sig = if (Scip.field(ix.infos.items[symbol], 7)) |f| (Scip.field(f.bytes, 5) orelse Scip.Field{ .num = 0 }).bytes else ""; + const note = if (!named) "" else if (sig.len == 0) displayName(ix, symbol) else try std.fmt.allocPrint(cx.arena, "{s} {s}", .{ displayName(ix, symbol), flat(cx.arena, sig) }); + try emitOcc(cx, ix, o, note); + }; +} + +fn emitOcc(cx: *Cx, ix: *const Scip.Index, o: u32, note: []const u8) Err!void { + const occ = ix.occurrences[o]; + try emitPath(cx, ix.documents[occ.doc].enc, try indexPath(cx.arena, ix, occ.doc), occ.range, note); +} + +/// The symbols `symbol`'s relationships name with flag `flag` set +/// (scip.proto Relationship: 2 is_reference, 3 is_implementation, 4 +/// is_type_definition), the ones the index knows. +fn related(a: std.mem.Allocator, ix: *const Scip.Index, symbol: u32, flag: u64) Err![]u32 { + var out: std.ArrayList(u32) = .empty; + var r: Scip.Reader = .{ .bytes = ix.infos.items[symbol] }; + while (r.next() catch null) |f| if (f.num == 4) { + const to = Scip.field(f.bytes, 1) orelse continue; + if ((Scip.field(f.bytes, flag) orelse continue).int == 0) continue; + if (ix.ids.get(to.bytes)) |id| try out.append(a, id); + }; + return out.items; +} + +/// A symbol's name to show: its display name, else its string's last +/// descriptor (`... lib/add().` is `add`). +fn displayName(ix: *const Scip.Index, symbol: u32) []const u8 { + if (Scip.field(ix.infos.items[symbol], 6)) |f| return f.bytes; + const s = std.mem.trimEnd(u8, ix.symbols.items[symbol], "().#/:!"); + return s[if (std.mem.lastIndexOfAny(u8, s, " /#.")) |cut| cut + 1 else 0..]; +} + +fn indexPath(a: std.mem.Allocator, ix: *const Scip.Index, doc: u32) Err![]const u8 { + return std.fmt.allocPrint(a, "{s}/{s}", .{ ix.root(), ix.documents[doc].path }); +} + +fn indexUri(a: std.mem.Allocator, ix: *const Scip.Index, doc: u32) Err![]const u8 { + var b: std.ArrayList(u8) = .empty; + try uriOf(&b, a, try indexPath(a, ix, doc)); + return b.items; +} + +/// The Index word's command line: the language's indexer, at the +/// project's root, where the index it writes answers for every file under +/// it (`PARDES_SCIP_RS` and the like name another indexer). +fn indexCommand(req: lsp.Req, out: *std.Io.Writer) !void { + const si = specFor(req.path) orelse return lsp.none(out, "no indexer for {s}", .{std.fs.path.basename(req.path)}); + const cmd = if (libc.getenv(indexer_envs[si])) |v| std.mem.span(v) else specs[si].indexer; + try out.writeAll("@run cd '"); + for (rootOf(std.fs.path.dirname(req.path) orelse "/", specs[si].markers)) |ch| try if (ch == '\'') out.writeAll("'\\''") else out.writeByte(ch); + try out.print("' && {s}\n", .{cmd}); +} + // ------------------------------------------------------------ row rendering /// Files read while answering ONE query, so a references list over a handful @@ -1283,8 +1527,11 @@ fn emitRange(cx: *Cx, enc: Enc, uri: []const u8, range: ?std.json.Value, note: [ } fn emitAt(cx: *Cx, enc: Enc, uri: []const u8, r: Range, note: []const u8) Err!void { + return emitPath(cx, enc, pathOf(cx.arena, uri) orelse return, r, note); +} + +fn emitPath(cx: *Cx, enc: Enc, path: []const u8, r: Range, note: []const u8) Err!void { if (cx.rows >= max_rows) return; - const path = pathOf(cx.arena, uri) orelse return; if (path.len == 0 or path[0] != '/') return; // rows promise absolute-or-rel-from-base // Without sources (Callstack) the asking file is still known: its rows' // columns convert from utf-16 as ever. @@ -1327,7 +1574,7 @@ fn colBytes(lntext: []const u8, ch: u32, enc: Enc) usize { var i: usize = 0; while (i < lntext.len and units < ch) { const l = std.unicode.utf8ByteSequenceLength(lntext[i]) catch 1; - units += if (l == 4) 2 else 1; + units += if (l == 4 and enc == .utf16) 2 else 1; i += l; } return i; @@ -1374,7 +1621,7 @@ fn strictOffset(src: []const u8, line: u32, ch: u32, enc: Enc) ?usize { var i: usize = bol; while (i < eol and units < ch) { const l = std.unicode.utf8ByteSequenceLength(src[i]) catch 1; - units += if (l == 4) 2 else 1; + units += if (l == 4 and enc == .utf16) 2 else 1; i += l; } if (units < ch) return null; @@ -2135,7 +2382,28 @@ fn notification(c: *Conn, io: std.Io, si: usize, g: u32, raw: []const u8, method // ------------------------------------------------------------- introspection -fn status(io: std.Io, out: *std.Io.Writer) !void { +fn status(io: std.Io, path: []const u8, out: *std.Io.Writer) !void { + // The index the file would ask first, what it holds and how old it is. + const dir = std.fs.path.dirname(path) orelse "/"; + if (Scip.find(io, dir)) |found| if (found) |ix| { + var symbols: usize = 0; + for (ix.infos.items) |info| symbols += @intFromBool(info.len > 0); + var newer: usize = 0; + var pbuf: [4096]u8 = undefined; + for (ix.documents) |d| newer += @intFromBool(ix.stale(io, std.fmt.bufPrint(&pbuf, "{s}/{s}", .{ ix.root(), d.path }) catch continue)); + try out.print("scip: {s}, {d} KB, {d} documents, {d} symbols", .{ ix.path(), (ix.size + 999) / 1000, ix.documents.len, symbols }); + if (!hideTime()) { + const secs: std.time.epoch.EpochSeconds = .{ .secs = @intCast(@divTrunc(ix.mtime, std.time.ns_per_s)) }; + const day = secs.getEpochDay().calculateYearDay(); + const md = day.calculateMonthDay(); + try out.print(", indexed {d}-{d:0>2}-{d:0>2} {d:0>2}:{d:0>2} UTC", .{ day.year, md.month.numeric(), md.day_index + 1, secs.getDaySeconds().getHoursIntoDay(), secs.getDaySeconds().getMinutesIntoHour() }); + } + try out.print(", {d} files newer\n", .{newer}); + } else { + try out.print("scip: no index.scip above {s}", .{dir}); + if (specFor(path)) |si| try out.print("; Index runs {s}", .{if (libc.getenv(indexer_envs[si])) |v| std.mem.span(v) else specs[si].indexer}); + try out.writeByte('\n'); + } else |e| try out.print("scip: index.scip above {s}: {s}\n", .{ dir, @errorName(e) }); try out.print("language servers:\n", .{}); for (&specs, 0..) |*s, i| { const c = &conns[i]; @@ -2178,12 +2446,12 @@ fn status(io: std.Io, out: *std.Io.Writer) !void { shown += 1; if (hideTime()) { try out.print(" {s:<18} {s:<14} {s:<20} {d:>4} row(s){s}{s}\n", .{ - @tagName(e.kind), specs[e.spec].name, e.file[0..e.file_len], e.rows, + @tagName(e.kind), sourceName(e.spec), e.file[0..e.file_len], e.rows, if (e.err_len > 0) " ERROR " else "", e.err[0..e.err_len], }); } else { try out.print(" {s:<18} {s:<14} {s:<20} {d:>7}us {d:>4} row(s){s}{s}\n", .{ - @tagName(e.kind), specs[e.spec].name, e.file[0..e.file_len], e.us, e.rows, + @tagName(e.kind), sourceName(e.spec), e.file[0..e.file_len], e.us, e.rows, if (e.err_len > 0) " ERROR " else "", e.err[0..e.err_len], }); } @@ -2453,7 +2721,7 @@ fn posOf(src: []const u8, off: u32, enc: Enc) Pos { var i: usize = bol; while (i < bol + lc.col) { const l = std.unicode.utf8ByteSequenceLength(src[i]) catch 1; - units += if (l == 4) 2 else 1; + units += if (l == 4 and enc == .utf16) 2 else 1; i += l; } return .{ .line = @intCast(lc.line), .ch = units }; @@ -2484,7 +2752,7 @@ fn items(v: ?std.json.Value) []std.json.Value { return if (o == .array) o.array.items else &.{}; } -const Range = struct { sl: u32, sc: u32, el: u32, ec: u32 }; +const Range = Scip.Range; fn rangeOf(range: ?std.json.Value) ?Range { const st = get(range, "start") orelse return null; @@ -2513,7 +2781,7 @@ test "caller trees reverse incoming paths and merge shared callers and sites" { defer arena_state.deinit(); const arena = arena_state.allocator(); const fixture = try std.json.parseFromSliceLeaky(std.json.Value, arena, "[{\"from\":{\"name\":\"left\",\"uri\":\"file:///test.c\",\"selectionRange\":{\"start\":{\"line\":3,\"character\":0},\"end\":{\"line\":3,\"character\":4}}},\"fromRanges\":[{\"start\":{\"line\":3,\"character\":0},\"end\":{\"line\":3,\"character\":4}}]},{\"from\":{\"name\":\"right\",\"uri\":\"file:///test.c\",\"selectionRange\":{\"start\":{\"line\":4,\"character\":0},\"end\":{\"line\":4,\"character\":4}}},\"fromRanges\":[{\"start\":{\"line\":4,\"character\":0},\"end\":{\"line\":4,\"character\":4}}]},{\"from\":{\"name\":\"page\",\"uri\":\"file:///test.c\",\"selectionRange\":{\"start\":{\"line\":1,\"character\":0},\"end\":{\"line\":1,\"character\":4}}},\"fromRanges\":[{\"start\":{\"line\":1,\"character\":0},\"end\":{\"line\":1,\"character\":4}}]},{\"from\":{\"name\":\"page\",\"uri\":\"file:///test.c\",\"selectionRange\":{\"start\":{\"line\":1,\"character\":0},\"end\":{\"line\":1,\"character\":4}}},\"fromRanges\":[{\"start\":{\"line\":2,\"character\":0},\"end\":{\"line\":2,\"character\":4}}]},{\"from\":{\"name\":\"main\",\"uri\":\"file:///test.c\",\"selectionRange\":{\"start\":{\"line\":0,\"character\":0},\"end\":{\"line\":0,\"character\":4}}},\"fromRanges\":[{\"start\":{\"line\":0,\"character\":0},\"end\":{\"line\":0,\"character\":4}}]}]", .{}); - const entries = items(fixture); + const entries = try serverHops(arena, fixture); var tree: CallTree = .{}; try tree.addPath(arena, &.{ entries[0], entries[2], entries[4] }, false); try tree.addPath(arena, &.{ entries[1], entries[3], entries[4] }, false); @@ -2552,9 +2820,9 @@ test "test callers sort after the rest, each group by place" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); - const fixture = try std.json.parseFromSliceLeaky(std.json.Value, arena, "[{\"name\":\"test b\",\"uri\":\"file:///a.zig\",\"selectionRange\":{\"start\":{\"line\":0,\"character\":0},\"end\":{\"line\":0,\"character\":1}}},{\"name\":\"z\",\"uri\":\"file:///b.zig\",\"selectionRange\":{\"start\":{\"line\":0,\"character\":0},\"end\":{\"line\":0,\"character\":1}}},{\"name\":\"x\",\"uri\":\"file:///a_test.zig\",\"selectionRange\":{\"start\":{\"line\":0,\"character\":0},\"end\":{\"line\":0,\"character\":1}}},{\"name\":\"y\",\"uri\":\"file:///a.zig\",\"selectionRange\":{\"start\":{\"line\":5,\"character\":0},\"end\":{\"line\":5,\"character\":1}}}]", .{}); + const fixture = try std.json.parseFromSliceLeaky(std.json.Value, arena, "[{\"from\":{\"name\":\"test b\",\"uri\":\"file:///a.zig\",\"selectionRange\":{\"start\":{\"line\":0,\"character\":0},\"end\":{\"line\":0,\"character\":1}}}},{\"from\":{\"name\":\"z\",\"uri\":\"file:///b.zig\",\"selectionRange\":{\"start\":{\"line\":0,\"character\":0},\"end\":{\"line\":0,\"character\":1}}}},{\"from\":{\"name\":\"x\",\"uri\":\"file:///a_test.zig\",\"selectionRange\":{\"start\":{\"line\":0,\"character\":0},\"end\":{\"line\":0,\"character\":1}}}},{\"from\":{\"name\":\"y\",\"uri\":\"file:///a.zig\",\"selectionRange\":{\"start\":{\"line\":5,\"character\":0},\"end\":{\"line\":5,\"character\":1}}}}]", .{}); var tree: CallTree = .{}; - for (items(fixture)) |item| try tree.nodes.append(arena, .{ .call = try callOf(arena, item) }); + for (try serverHops(arena, fixture)) |hop| try tree.nodes.append(arena, .{ .call = hop.call }); var order = [_]usize{ 0, 1, 2, 3 }; std.mem.sort(usize, &order, @as(*const CallTree, &tree), callerLess); try std.testing.expectEqualSlices(usize, &.{ 3, 1, 0, 2 }, &order); @@ -2810,7 +3078,7 @@ test "a language with no server to start says so, not that it did not answer" { test "Lspinfo before any query says a server is not started yet, not off" { var out: std.Io.Writer.Allocating = .init(std.testing.allocator); defer out.deinit(); - try status(std.testing.io, &out.writer); + try status(std.testing.io, "/x/notes.txt", &out.writer); try std.testing.expect(std.mem.indexOf(u8, out.written(), "not started yet (its first query starts it)") != null); } diff --git a/src/pardes.zig b/src/pardes.zig index 88045147..e65e48eb 100644 --- a/src/pardes.zig +++ b/src/pardes.zig @@ -6623,6 +6623,7 @@ pub const Pardes = struct { .subtypes => "Subtypes", .rename => "Rename", .format => "Format", + .index => "Index", else => @tagName(kind), }; } @@ -6661,6 +6662,12 @@ pub const Pardes = struct { const why = std.mem.trimEnd(u8, rows["@none ".len..], "\n"); return p.reportFailure(w.pane, std.fmt.bufPrint(&said, "{s}: {s}", .{ lspWord(w.kind), why[0..@min(why.len, 200)] }) catch "no answer"); } + // Index's answer is its indexer's command line, run in a command + // pane (lsp_client.indexCommand). + if (w.kind == .index) { + if (std.mem.startsWith(u8, rows, "@run ")) _ = exec.execute(p, w.pane, std.mem.trimEnd(u8, rows["@run ".len..], "\n")); + return; + } if (w.kind == .rename or w.kind == .format) { const fallback: ?[]const u8 = if (w.kind == .rename) w.arg.slice() else null; if (p.parseLspEdits(rows, fallback)) |edits| { diff --git a/test/behaviour.zig b/test/behaviour.zig index 8f412df6..2e632911 100644 --- a/test/behaviour.zig +++ b/test/behaviour.zig @@ -12,6 +12,7 @@ test { _ = @import("behaviour/flags.zig"); _ = @import("behaviour/from_acme.zig"); _ = @import("behaviour/keys.zig"); + _ = @import("behaviour/language_servers.zig"); _ = @import("behaviour/limits.zig"); _ = @import("behaviour/log.zig"); _ = @import("behaviour/look_and_exec.zig"); diff --git a/test/behaviour/language_servers.zig b/test/behaviour/language_servers.zig new file mode 100644 index 00000000..da7b3d59 --- /dev/null +++ b/test/behaviour/language_servers.zig @@ -0,0 +1,294 @@ +//! docs/typ/setup.typ, Language servers <language-servers>: a file's +//! language words ask the nearest `index.scip` above it first (SCIP, +//! read by src/lsp/Scip.zig) and its language server when the index has +//! nothing to say, here the scripted one (test/lspmock.zig). The indexes +//! are made by the protobuf writer below, but for the one a real indexer +//! wrote (test/scipfixture). +const std = @import("std"); +const Session = @import("session").Session; + +const lib_rs = + \\pub trait Shape { + \\ fn area(&self) -> u32; + \\} + \\/// Adds two numbers. + \\pub fn add(a: u32, b: u32) -> u32 { + \\ a + b + \\} + \\pub struct Square; + \\impl Shape for Square { + \\ fn area(&self) -> u32 { add(2, 2) } + \\} + \\ +; +const main_rs = + \\mod lib; + \\use lib::add; + \\fn main() { + \\ let x = add(1, 2); + \\ let _ = x; + \\} + \\ +; + +/// One occurrence: its document (0 main.rs, 1 lib.rs), line and columns, +/// symbol, roles, and the node a definition names. +const Occ = struct { doc: u8, line: u32, from: u32, to: u32, sym: []const u8, roles: u32 = 0, node: ?[4]u32 = null }; +/// One symbol's information, in the document defining it. +const Sym = struct { doc: u8, sym: []const u8, kind: u32, name: []const u8, sig: []const u8 = "", docs: []const u8 = "", implements: []const u8 = "" }; + +const occs = [_]Occ{ + .{ .doc = 0, .line = 1, .from = 9, .to = 12, .sym = "t . add().", .roles = 2 }, + .{ .doc = 0, .line = 2, .from = 3, .to = 7, .sym = "t . main().", .roles = 1, .node = .{ 2, 0, 5, 1 } }, + .{ .doc = 0, .line = 3, .from = 8, .to = 9, .sym = "local 0", .roles = 1 }, + .{ .doc = 0, .line = 3, .from = 12, .to = 15, .sym = "t . add()." }, + .{ .doc = 0, .line = 4, .from = 12, .to = 13, .sym = "local 0" }, + .{ .doc = 1, .line = 0, .from = 10, .to = 15, .sym = "t . Shape#", .roles = 1, .node = .{ 0, 0, 2, 1 } }, + .{ .doc = 1, .line = 1, .from = 7, .to = 11, .sym = "t . Shape#area().", .roles = 1, .node = .{ 1, 4, 1, 26 } }, + .{ .doc = 1, .line = 4, .from = 7, .to = 10, .sym = "t . add().", .roles = 1, .node = .{ 3, 0, 6, 1 } }, + .{ .doc = 1, .line = 4, .from = 11, .to = 12, .sym = "local 0", .roles = 1 }, + .{ .doc = 1, .line = 7, .from = 11, .to = 17, .sym = "t . Square#", .roles = 1, .node = .{ 7, 0, 7, 18 } }, + .{ .doc = 1, .line = 8, .from = 5, .to = 10, .sym = "t . Shape#" }, + .{ .doc = 1, .line = 8, .from = 15, .to = 21, .sym = "t . Square#" }, + .{ .doc = 1, .line = 9, .from = 7, .to = 11, .sym = "t . Square#area().", .roles = 1, .node = .{ 9, 4, 9, 39 } }, + .{ .doc = 1, .line = 9, .from = 28, .to = 31, .sym = "t . add()." }, +}; +const syms = [_]Sym{ + .{ .doc = 0, .sym = "t . main().", .kind = 17, .name = "main", .sig = "fn main()" }, + .{ .doc = 0, .sym = "local 0", .kind = 61, .name = "x", .sig = "let x: u32" }, + .{ .doc = 1, .sym = "t . Shape#", .kind = 53, .name = "Shape", .sig = "pub trait Shape" }, + .{ .doc = 1, .sym = "t . Shape#area().", .kind = 70, .name = "area", .sig = "fn area(&self) -> u32" }, + .{ .doc = 1, .sym = "t . add().", .kind = 17, .name = "add", .sig = "pub fn add(a: u32, b: u32) -> u32", .docs = "Adds two numbers." }, + .{ .doc = 1, .sym = "local 0", .kind = 37, .name = "a", .sig = "a: u32" }, + .{ .doc = 1, .sym = "t . Square#", .kind = 49, .name = "Square", .implements = "t . Shape#" }, + .{ .doc = 1, .sym = "t . Square#area().", .kind = 26, .name = "area", .sig = "fn area(&self) -> u32", .implements = "t . Shape#area()." }, +}; + +/// A protobuf message, its fields appended in order. +const Msg = struct { + b: std.ArrayList(u8) = .empty, + + fn varint(m: *Msg, a: std.mem.Allocator, v: u64) void { + var x = v; + while (x >= 0x80) : (x >>= 7) m.b.append(a, @as(u8, @truncate(x)) | 0x80) catch @panic("oom"); + m.b.append(a, @truncate(x)) catch @panic("oom"); + } + + fn int(m: *Msg, a: std.mem.Allocator, num: u64, v: u64) void { + m.varint(a, num << 3); + m.varint(a, v); + } + + fn bytes(m: *Msg, a: std.mem.Allocator, num: u64, v: []const u8) void { + m.varint(a, num << 3 | 2); + m.varint(a, v.len); + m.b.appendSlice(a, v) catch @panic("oom"); + } +}; + +/// The index of main.rs and lib.rs: typed ranges, as new indexers write +/// them (single-line ranges, a definition's node a multi-line one). +fn index(a: std.mem.Allocator) []const u8 { + var ix: Msg = .{}; + for ([_][]const u8{ "main.rs", "lib.rs" }, 0..) |path, d| { + var doc: Msg = .{}; + doc.bytes(a, 1, path); + for (occs) |o| if (o.doc == d) { + var m: Msg = .{}; + var r: Msg = .{}; + for ([_]u64{ o.line, o.from, o.to }, 1..) |v, n| r.int(a, n, v); + m.bytes(a, 8, r.b.items); + m.bytes(a, 2, o.sym); + if (o.roles != 0) m.int(a, 3, o.roles); + if (o.node) |node| { + var e: Msg = .{}; + for (node, 1..) |v, n| e.int(a, n, v); + m.bytes(a, 11, e.b.items); + } + doc.bytes(a, 2, m.b.items); + }; + for (syms) |s| if (s.doc == d) { + var m: Msg = .{}; + m.bytes(a, 1, s.sym); + if (s.docs.len > 0) m.bytes(a, 3, s.docs); + if (s.implements.len > 0) { + var rel: Msg = .{}; + rel.bytes(a, 1, s.implements); + rel.int(a, 3, 1); + m.bytes(a, 4, rel.b.items); + } + m.int(a, 5, s.kind); + m.bytes(a, 6, s.name); + if (s.sig.len > 0) { + var sig: Msg = .{}; + sig.bytes(a, 5, s.sig); + m.bytes(a, 7, sig.b.items); + } + doc.bytes(a, 3, m.b.items); + }; + doc.int(a, 6, 1); + ix.bytes(a, 2, doc.b.items); + } + return ix.b.items; +} + +/// main.rs, lib.rs and their index in a directory of their own, main.rs +/// shown with its cursor at `at` (`4:13`, its call of add); its serial. +fn open(s: *Session, tmp: *std.testing.TmpDir, at: []const u8) !u32 { + const io = std.testing.io; + try tmp.dir.writeFile(io, .{ .sub_path = "main.rs", .data = main_rs }); + try tmp.dir.writeFile(io, .{ .sub_path = "lib.rs", .data = lib_rs }); + try tmp.dir.writeFile(io, .{ .sub_path = "index.scip", .data = index(s.arena.allocator()) }); + var buf: [4096]u8 = undefined; + const dir = buf[0..try tmp.dir.realPath(io, &buf)]; + return s.lookFile(s.at("{s}/main.rs:{s}", .{ dir, at })); +} + +fn has(hay: []const u8, needle: []const u8) bool { + return std.mem.indexOf(u8, hay, needle) != null; +} + +/// The body of the pane /index names with `suffix` (`+Hover`). +fn bodyOf(s: *Session, suffix: []const u8) ![]const u8 { + var rows = std.mem.splitScalar(u8, try s.read("/index"), '\n'); + while (rows.next()) |line| { + // serial kind dirty name column + const name = line[0 .. std.mem.lastIndexOfScalar(u8, line, ' ') orelse continue]; + if (std.mem.endsWith(u8, name, suffix)) return s.read(s.at("/pane/{s}/body", .{line[0..std.mem.indexOfScalar(u8, line, ' ').?]})); + } + return error.NoSuchPane; +} + +test "language-servers: Def asks the index first, and lands on the definition it names" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + const main = try open(&s, &tmp, "4:13"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Def\n"); + const lib = try s.focused(); + try std.testing.expect(std.mem.endsWith(u8, try s.read(s.at("/pane/{d}/name", .{lib})), "/lib.rs\n")); + // `add` on lib.rs's fifth line, where the scripted server would have + // said its second. + try std.testing.expectEqual(.{ @as(u32, 76), @as(u32, 79) }, try s.range(s.at("/pane/{d}/dot", .{lib}))); +} + +test "language-servers: Refs lists every occurrence the index has, the import and the definition among them" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + const main = try open(&s, &tmp, "4:13"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Refs\n"); + const rows = try s.read(s.at("/pane/{d}/body", .{try s.focused()})); + for ([_][]const u8{ "main.rs:2:10 ", "main.rs:4:13 ", "lib.rs:5:8 ", "lib.rs:10:29 " }) |place| + try std.testing.expect(has(rows, place)); +} + +test "language-servers: Hover says the signature and the documentation the index has" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + const main = try open(&s, &tmp, "4:13"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Hover\n"); + try std.testing.expectEqualStrings("pub fn add(a: u32, b: u32) -> u32\n\nAdds two numbers.\n", try bodyOf(&s, "+Hover")); +} + +test "language-servers: Symbols lists the file's definitions by name, a local's left out" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + const main = try open(&s, &tmp, "4:13"); + try s.write(s.at("/pane/{d}/look", .{main}), s.at("{s}/lib.rs\n", .{std.fs.path.dirname(std.mem.trimEnd(u8, try s.read(s.at("/pane/{d}/name", .{main})), "\n")).?})); + try s.write(s.at("/pane/{d}/ctl", .{try s.focused()}), "Symbols\n"); + const rows = try bodyOf(&s, "+Search"); + for ([_][]const u8{ "lib.rs:1:11-15\nShape\n", "lib.rs:2:8-11\narea\n", "lib.rs:5:8-10\nadd\n", "lib.rs:8:12-17\nSquare\n", "lib.rs:10:8-11\narea\n" }) |row| + try std.testing.expect(has(rows, row)); + try std.testing.expect(!has(rows, "lib.rs:5:12")); +} + +test "language-servers: Callers lists each call with the function it is made from, Subtypes the implementations the index relates" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + const main = try open(&s, &tmp, "4:13"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Callers\n"); + const rows = try bodyOf(&s, "+Search"); + try std.testing.expect(has(rows, "main.rs:4:13-15\nmain\n")); + try std.testing.expect(has(rows, "lib.rs:10:29-31\narea\n")); + // Shape's one implementation is Square: a jump there, as gi's. + const lib = try s.lookFile(s.at("{s}/lib.rs:1:11", .{std.fs.path.dirname(std.mem.trimEnd(u8, try s.read(s.at("/pane/{d}/name", .{main})), "\n")).?})); + try s.write(s.at("/pane/{d}/ctl", .{lib}), "Subtypes\n"); + try std.testing.expectEqual(.{ @as(u32, 128), @as(u32, 134) }, try s.range(s.at("/pane/{d}/dot", .{try s.focused()}))); +} + +test "language-servers: what the index does not know, and a pane with unsaved changes, ask the server" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + // `mod`, which no occurrence covers: the scripted server's answer, + // lib.rs's second line. + const main = try open(&s, &tmp, "1:1"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Def\n"); + try std.testing.expectEqual(.{ @as(u32, 25), @as(u32, 28) }, try s.range(s.at("/pane/{d}/dot", .{try s.focused()}))); + try s.write(s.at("/pane/{d}/addr", .{main}), "4:13"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "dot=addr\n"); + try s.write(s.at("/pane/{d}/data", .{main}), ""); + try s.write(s.at("/pane/{d}/addr", .{main}), "$"); + try s.write(s.at("/pane/{d}/data", .{main}), "// unsaved\n"); + try s.write(s.at("/pane/{d}/addr", .{main}), "4:13"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "dot=addr\n"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Def\n"); + try std.testing.expectEqual(.{ @as(u32, 25), @as(u32, 28) }, try s.range(s.at("/pane/{d}/dot", .{try s.focused()}))); +} + +test "language-servers: a real indexer's index answers too (rust-analyzer's, test/scipfixture), and Lspinfo says what it holds" { + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + const main = try s.lookFile("test/scipfixture/main.rs:5:13"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Hover\n"); + try std.testing.expectEqualStrings("pub fn add(a: i32, b: i32) -> i32\n\nAdds two numbers.\n", try bodyOf(&s, "+Hover")); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Def\n"); + try std.testing.expect(std.mem.endsWith(u8, try s.read(s.at("/pane/{d}/name", .{try s.focused()})), "/test/scipfixture/lib.rs\n")); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Lspinfo\n"); + const info = try bodyOf(&s, "+Lsp"); + try std.testing.expect(std.mem.startsWith(u8, info, "scip: ")); + try std.testing.expect(has(info, "/test/scipfixture/index.scip, 2 KB, 2 documents, 7 symbols")); +} + +test "language-servers: Callstack's tree is the index's, each caller the definition holding its call, a function nobody calls a leaf" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + const main = try open(&s, &tmp, "4:13"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Callstack\n"); + try s.expect(s.at("/pane/{d}/body", .{try s.focused()}), + \\├──┬ lib.rs:10:8-11 + \\│ │ fn area(&self) -> u32 { add(2, 2) } + \\│ ╰─ lib.rs:10:29-31 + \\│ fn area(&self) -> u32 { add(2, 2) } + \\╰──┬ main.rs:3:4-7 + \\ │ fn main() { + \\ ╰─ main.rs:4:13-15 + \\ let x = add(1, 2); + \\ + ); +} + +test "language-servers: Index runs the language's indexer at the project's root, in a command pane" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var s = try Session.initLanguage(std.testing.allocator); + defer s.deinit(); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "Cargo.toml", .data = "[package]\n" }); + const main = try open(&s, &tmp, "1:1"); + try s.write(s.at("/pane/{d}/ctl", .{main}), "Index\n"); + var buf: [4096]u8 = undefined; + const dir = buf[0..try tmp.dir.realPath(std.testing.io, &buf)]; + try std.testing.expect(std.mem.startsWith(u8, try bodyOf(&s, dir), s.at("% cd '{s}' && rust-analyzer scip .", .{dir}))); +} diff --git a/test/process/builtins.zig b/test/process/builtins.zig index 55f4ed49..b3dbe05e 100644 --- a/test/process/builtins.zig +++ b/test/process/builtins.zig @@ -7,7 +7,7 @@ const has = process.has; const check = process.check; pub const cases = .{ - .{ "builtins: Lspinfo opens one +Lsp pane saying its language servers, detached and in a pty", lspinfo }, + .{ "builtins: Lspinfo opens one +Lsp pane saying its index and its language servers, detached and in a pty", lspinfo }, }; fn lspinfo(gpa: std.mem.Allocator, exe: [:0]const u8) !void { @@ -43,7 +43,7 @@ fn lspinfo(gpa: std.mem.Allocator, exe: [:0]const u8) !void { process.nap(5); }; const body = try c.pane(a, lsp, "body"); - try check(std.mem.startsWith(u8, body, "language servers:\n") and has(body, " zls "), "{s}", .{body}); + try check(std.mem.startsWith(u8, body, "scip: no index.scip above ") and has(body, "\nlanguage servers:\n") and has(body, " zls "), "{s}", .{body}); try std.testing.expectEqualStrings("initial\n", try c.read(a, "/pane/1/body")); var f: process.Files = .init(gpa, address); defer f.deinit(); diff --git a/test/scipfixture/Cargo.toml b/test/scipfixture/Cargo.toml new file mode 100644 index 00000000..b981003b --- /dev/null +++ b/test/scipfixture/Cargo.toml @@ -0,0 +1,11 @@ +# A crate rust-analyzer indexed once (`rust-analyzer scip .`): its +# index.scip is the real indexer's output the T1 language cases read +# (test/behaviour/language_servers.zig), beside their hand-made ones. +[package] +name = "scipfixture" +version = "0.0.0" +edition = "2021" + +[[bin]] +name = "scipfixture" +path = "main.rs" diff --git a/test/scipfixture/index.scip b/test/scipfixture/index.scip Binary files differnew file mode 100644 index 00000000..f9136e66 --- /dev/null +++ b/test/scipfixture/index.scip diff --git a/test/scipfixture/lib.rs b/test/scipfixture/lib.rs new file mode 100644 index 00000000..a7408596 --- /dev/null +++ b/test/scipfixture/lib.rs @@ -0,0 +1,4 @@ +/// Adds two numbers. +pub fn add(a: i32, b: i32) -> i32 { + a + b +} diff --git a/test/scipfixture/main.rs b/test/scipfixture/main.rs new file mode 100644 index 00000000..39adebaa --- /dev/null +++ b/test/scipfixture/main.rs @@ -0,0 +1,7 @@ +mod lib; +use lib::add; + +fn main() { + let x = add(1, 2); + let _ = x; +} |
