From fa82cac885cb4738fe36d1e49b4749b5a3e31a4a Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Sun, 6 Sep 2026 17:14:40 -0300 Subject: syntax: a results buffer is coloured as the code it quotes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit +Grep, +Search and every language answer render rows like src/look.zig:718:12-16 fn grepText(path: []const u8, text: []const u8... — a location, a space, and a piece of some file. The location names the file, the file names the grammar, and the rest of the row is a fragment of that language, so a grep over Zig reads as Zig and one Markdown row in the same buffer does not pretend otherwise. The location itself is left uncoloured: it is not code, and painting it as code is how a path starts looking like a keyword. `look.parsePathLine` decides what counts as a location — the same primitive n/N already walks these buffers with, so the two agree by construction about which rows are locations. NOT `lookableLineSpan`, which is n/N's whole heuristic: it calls `resolve`, and a `realpath` per row per scroll is not something a render path can afford. A bare filename is refused too; only `path:line` counts, or a prose line whose first word ended in `.md` would colour the rest of a sentence. THE BUFFER IS COLOURED WHOLE, ONCE, WHEN IT IS FILLED. An adversarial pass measured the obvious per-window implementation and it was untenable: the rows are independent, so a window pass buys no fidelity, only amortisation, and pays a burst on every scroll that outran the covered range. Grammars compile their highlights query on first use — zig 26.9ms, cpp 18.9ms, rust 14.3ms — so a polyglot grep showing six languages stalled a frame by 66ms, moving a cost the syntax module had deliberately put on "opening a file" onto a scroll. It also raised tree-sitter's allocation rate 3.5x (8,785 per refresh against 2,454) into a 16 MiB bump arena that only reclaims LIFO, so ~16 scroll re-highlights exhausted it — and that arena is shared with real file panes, so a results pane could evict editing. A grep is capped at 512 rows; colouring it once makes the covered-range check true forever after and scrolling free. The rest of that pass, in the same spirit: injections off for a single row (both build a SECOND parser, per fenced block and per inline node, which is absurd for one truncated row that almost never contains a fence), one query cursor for the buffer instead of one per row, a one-entry extension memo so non-matching rows stop paying a 29-spec scan, and NO highlights at all when nothing painted — an all-zero run is not the same as none, and it defeated `recolorSyntax`'s fast path, making every +Help and +Config walk its graphemes every frame to paint nothing. One correctness bug from the same pass: a failed `setLanguage` has already nulled the parser's language, so leaving `held` on the previous grammar made every later row of it skip the call and silently lose colour. Documents keep `.source`: a New scratch and a real file are output-shaped but have one language and an edit per keystroke, and `saves` is the line between the two. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016Q4RATpafkwahrovHQLKRf --- src/syntax.zig | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 209 insertions(+), 9 deletions(-) (limited to 'src/syntax.zig') diff --git a/src/syntax.zig b/src/syntax.zig index 09ff12cc..f6e7cae2 100644 --- a/src/syntax.zig +++ b/src/syntax.zig @@ -6,6 +6,7 @@ const std = @import("std"); const config = @import("pardes_config"); const tracy = @import("tracy.zig"); const grammar_manifest = @import("grammar_manifest.zig"); +const look = @import("look.zig"); pub const enabled = config.syntax_highlighting; const zig_grammar = config.syntax_zig_grammar; @@ -226,17 +227,19 @@ fn forLang(name: []const u8) !?Selected { return null; } -fn runQuery(styles: []u8, sel: Selected, tree: *ts.Tree, base: usize) void { - const cursor = ts.QueryCursor.create(); - defer cursor.destroy(); +/// The caller owns the cursor: `ts_query_cursor_exec` fully resets its state, +/// so one cursor serves any number of trees, and the per-row pass would +/// otherwise create and destroy one — three allocations against the shared +/// tree-sitter arena — for every row of a results buffer. +fn runQuery(styles: []u8, sel: Selected, tree: *ts.Tree, base: usize, cursor: *ts.QueryCursor) void { cursor.exec(sel.query, tree.rootNode()); while (cursor.nextMatch()) |match| { for (match.captures) |cap| { const syn = synFor(sel.query.captureNameForId(cap.index) orelse ""); if (syn == .none) continue; - var b: usize = base + cap.node.startByte(); + const b = @min(base + cap.node.startByte(), styles.len); const end = @min(base + @as(usize, cap.node.endByte()), styles.len); - while (b < end) : (b += 1) styles[b] = @intFromEnum(syn); + if (end > b) @memset(styles[b..end], @intFromEnum(syn)); } } } @@ -307,18 +310,140 @@ pub fn highlightFileRange(gpa: std.mem.Allocator, path: []const u8, content: []c const parser = ts.Parser.create(); defer parser.destroy(); parser.setLanguage(selected.lang) catch return styles; - const tree = parser.parseString(source, null) orelse return styles; + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + paintWith(styles, source, selected, parser, cursor, true); + return styles; +} + +/// Parse `source` and write its style bytes into `styles`, with a parser and a +/// query cursor the caller owns, so the per-row pass below can run a whole +/// results buffer through one of each. +/// +/// `inject` is off for a single row. Both injection passes build a SECOND +/// parser of their own — per fenced block, per `inline` node — which is +/// amortised over a document and absurd over one truncated grep row that +/// almost never contains a fenced block to begin with. +fn paintWith( + styles: []u8, + source: []const u8, + selected: Selected, + parser: *ts.Parser, + cursor: *ts.QueryCursor, + inject: bool, +) void { + const tree = parser.parseString(source, null) orelse return; defer tree.destroy(); - runQuery(styles, selected, tree, 0); + runQuery(styles, selected, tree, 0, cursor); + if (!inject) return; if (InjectSite.forGrammar(selected.name)) |site| injectCodeBlocks(styles, source, tree.rootNode(), site); // Disjoint from the fenced-block pass above: `code_fence_content` is never // an `inline` node, so the two never write the same byte. if (std.mem.eql(u8, selected.name, "markdown")) injectMarkdownInline(styles, source, tree.rootNode()); +} + +/// A results buffer — every search, grep and language answer in this program — +/// coloured as the CODE it is quoting. +/// +/// The rows look like `src/look.zig:718:12-16 fn grepText(path: []const u8...`: +/// a location, a space, and a piece of some file. The location names the file, +/// the file names the grammar, and the rest of the row is a fragment of that +/// language — so a +Grep over Zig reads as Zig and one over Markdown does not +/// pretend to. `look.parsePathLine` decides what counts as a location, which is +/// the same primitive n/N walks these buffers with, so the two agree by +/// construction about which rows are locations. +/// +/// ONE PARSER AND ONE CURSOR for the whole buffer, and the buffer is coloured +/// once when it is filled rather than per visible window (file_pane +/// `refreshHighlights`) — the rows are independent, so a window pass buys no +/// fidelity and pays a burst of parses, cursors and first-time query compiles +/// on every scroll that outran the covered range. +pub fn highlightLocations(gpa: std.mem.Allocator, content: []const u8, start_byte_raw: usize, end_byte_raw: usize) ![]u8 { + const tz = tracy.zone(@src(), "highlightLocations"); + defer tz.end(); + const start_byte = @min(start_byte_raw, content.len); + const end_byte = @max(start_byte, @min(end_byte_raw, content.len)); + const source = content[start_byte..end_byte]; + if (!enabled) return &.{}; + const styles = try gpa.alloc(u8, source.len); + errdefer gpa.free(styles); + @memset(styles, 0); + + // Both are created on the first row that needs them and kept for the rest; + // `held` is the language the parser is currently set to. + var parser: ?*ts.Parser = null; + defer if (parser) |ptr| ptr.destroy(); + var cursor: ?*ts.QueryCursor = null; + defer if (cursor) |ptr| ptr.destroy(); + var held: ?Selected = null; + // Consecutive rows of a results buffer are overwhelmingly the same file, + // and `forExt` is a linear walk of 29 specs and their extension lists. One + // remembered answer collapses that to a string compare — including for the + // rows that match NOTHING (a jumplist `@p3:10`, a `.lock`, a `.txt`), + // which otherwise pay the whole failing scan every time. + var memo_ext: []const u8 = "\x00"; + var memo: ?Selected = null; + var painted = false; + + var offset: usize = 0; + var lines = std.mem.splitScalar(u8, source, '\n'); + while (lines.next()) |line| { + defer offset += line.len + 1; + const code = codeAfterLocation(line) orelse continue; + const ext = std.fs.path.extension(code.path); + if (!std.mem.eql(u8, ext, memo_ext)) { + memo_ext = ext; + memo = forExt(ext) catch null; + } + const selected = memo orelse continue; + if (parser == null) parser = ts.Parser.create(); + if (cursor == null) cursor = ts.QueryCursor.create(); + if (held == null or held.?.lang != selected.lang) { + // `held` is cleared FIRST: a failed `setLanguage` has already set + // the parser's language to null, so leaving `held` on the previous + // grammar makes every later row of it skip the call and parse + // against nothing — the rest of the buffer silently loses colour. + held = null; + parser.?.setLanguage(selected.lang) catch continue; + held = selected; + } + paintWith(styles[offset + code.at ..][0..code.text.len], code.text, selected, parser.?, cursor.?, false); + painted = true; + } + // NOTHING TO PAINT IS NOTHING TO KEEP. `recolorSyntax` skips a pane whose + // highlights are empty, and every output buffer without locations in it — + // +Help, +Config, +Messages, +Errors — would otherwise hand the renderer a + // full-length run of zeroes and make it walk every visible grapheme, every + // frame, to paint nothing. + if (!painted) { + gpa.free(styles); + return &.{}; + } return styles; } +/// The ` ` split of one results row, or null when the row is not +/// one. A row qualifies when its FIRST whitespace-delimited token is entirely a +/// look target — the whole token, so `see:` in prose does not count — and +/// something follows it. +fn codeAfterLocation(line: []const u8) ?struct { path: []const u8, at: usize, text: []const u8 } { + const token_end = std.mem.indexOfAny(u8, line, " \t") orelse return null; + if (token_end == 0) return null; + const token = line[0..token_end]; + const target = look.parsePathLine(token); + if (target.end != token.len) return null; + // A bare word is not a location: `main.zig` alone is a filename, but a + // results row is `main.zig:12:3`, and without that a prose line whose + // first word happens to end in `.md` would colour the rest of a sentence. + if (target.at.line == 0) return null; + var at = token_end; + while (at < line.len and (line[at] == ' ' or line[at] == '\t')) at += 1; + if (at >= line.len) return null; + return .{ .path = target.path, .at = at, .text = line[at..] }; +} + // Markdown fenced blocks and Typst raw blocks are the same construct — a // language tag plus a literal payload — under different node shapes, so one // walker drives both and only the (lang, content) extraction differs. @@ -399,7 +524,9 @@ fn highlightCodeBlock(styles: []u8, source: []const u8, block: ts.Node, site: In // outer colour would survive as a wash behind the injected code. The sub // grammar owns the payload outright: clear it first. @memset(styles[cs..ce], @intFromEnum(Syn.none)); - runQuery(styles, sub_sel, tree, cs); + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + runQuery(styles, sub_sel, tree, cs, cursor); } // tree-sitter-markdown is a split grammar: the block parser bottoms out at named @@ -445,7 +572,9 @@ fn highlightInline(styles: []u8, source: []const u8, node: ts.Node, sel: Selecte // the colour the heading must keep wherever the inline pass captures // nothing. Painting over instead of resetting is what makes *italic* inside // a heading recolour while the rest of the heading stays heading-coloured. - runQuery(styles, sel, tree, s); + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + runQuery(styles, sel, tree, s, cursor); } /// One Syn byte per byte of content[start, end) for unified diffs/patches. @@ -481,6 +610,77 @@ fn diffLineSyn(line: []const u8) Syn { return .none; } +test "a results row is coloured by the file its location names" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + const gpa = std.testing.allocator; + + // Two rows quoting two languages, plus a row that is not a location and a + // location with nothing after it. + const content = + "src/a.zig:1:1 const S = struct {};\n" ++ + "src/b.md:2:1 # heading\n" ++ + "just some prose with a colon: here\n" ++ + "src/c.zig:3:1\n"; + const styles = try highlightLocations(gpa, content, 0, content.len); + defer gpa.free(styles); + try std.testing.expectEqual(content.len, styles.len); + + // The LOCATION itself is left alone — it is not code, and colouring it as + // code is how a path starts looking like a keyword. + for (styles[0.."src/a.zig:1:1".len]) |b| try std.testing.expectEqual(@as(u8, 0), b); + + // ...and `struct` in the Zig row is a keyword, which is only true if the + // grammar was chosen from `a.zig` rather than from the buffer's own name. + // + // `struct` and not `const`: tree-sitter-zig captures `const` as + // `@type.qualifier`, which `synFor` maps to nothing — a real property of + // the shipped query rather than of this pass, and the reason the first + // version of this test failed. + const zig_kw = std.mem.indexOf(u8, content, "struct").?; + try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[zig_kw]); + try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[zig_kw + 5]); + + // A row with no location contributes nothing... + const prose = std.mem.indexOf(u8, content, "just some prose").?; + for (styles[prose .. prose + 14]) |b| try std.testing.expectEqual(@as(u8, 0), b); + // ...and neither does a location with no code after it. + const bare = std.mem.indexOf(u8, content, "src/c.zig").?; + for (styles[bare..]) |b| try std.testing.expectEqual(@as(u8, 0), b); +} + +test "a buffer with no locations in it keeps no highlights at all" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + // +Help, +Config, +Messages: prose. An all-zero run of styles is not the + // same as none — `recolorSyntax` skips a pane whose highlights are EMPTY, + // and returning a full-length run of zeroes made it walk every visible + // grapheme every frame to paint nothing. + const content = "nothing has been said yet\n0: save: AccessDenied (x2)\n"; + const styles = try highlightLocations(std.testing.allocator, content, 0, content.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(@as(usize, 0), styles.len); +} + +test "codeAfterLocation takes whole-token locations and nothing else" { + // A grep row: path, line, column range, then the quoted source. + const got = codeAfterLocation("src/x.zig:7:2-9 fn main() void {") orelse + return error.ShouldBeALocation; + try std.testing.expectEqualStrings("src/x.zig", got.path); + try std.testing.expectEqualStrings("fn main() void {", got.text); + + // Not locations: a bare filename (no line), prose with a colon, a token + // that only PARTLY parses, and a location with nothing after it. + try std.testing.expect(codeAfterLocation("main.zig some words") == null); + try std.testing.expect(codeAfterLocation("note: this is prose") == null); + try std.testing.expect(codeAfterLocation("src/x.zig:7:2x rest") == null); + try std.testing.expect(codeAfterLocation("src/x.zig:7:2") == null); + try std.testing.expect(codeAfterLocation("") == null); + try std.testing.expect(codeAfterLocation(" leading space") == null); +} + test "tree-sitter allocator callbacks preserve and free exact allocations" { syntax_allocator = std.testing.allocator; defer syntax_allocator = undefined; -- cgit v1.3