diff options
Diffstat (limited to 'src/syntax.zig')
| -rw-r--r-- | src/syntax.zig | 264 |
1 files changed, 243 insertions, 21 deletions
diff --git a/src/syntax.zig b/src/syntax.zig index 1494ebd3..09ff12cc 100644 --- a/src/syntax.zig +++ b/src/syntax.zig @@ -46,6 +46,34 @@ fn specCount() comptime_int { return count; } +// Upstream's typst highlights.scm names its markup honestly — +// @markup.heading.*, @markup.bold, @markup.italic, @markup.raw.block — and +// `synFor` now understands that vocabulary, so headings, bold, italics and raw +// blocks paint on their own. What is left here is exactly the two captures we +// refuse to map globally: +// +// - upstream tags call callees @function/@function.method; mapping "function" +// in `synFor` would recolour every function call in every language. +// - upstream tags the code sigil "#" @operator, and mapping "operator" would +// likewise light up every +, -, == in the codebase. +// +// Both are worth colouring *in typst specifically*: the sigil in front of every +// #let/#if/#import/#call is the visual anchor of the code/markup split, and +// upstream leaves it uncoloured even though the keyword behind it is not. +const typst_supplement = + \\ + \\(call item: (ident) @keyword) + \\(call item: (field field: (ident) @keyword)) + \\"#" @keyword + \\ +; + +fn querySrc(comptime g: grammar_manifest.Grammar) []const u8 { + const base = @field(ts_queries, g.name ++ "_highlights"); + if (comptime std.mem.eql(u8, g.name, "typst")) return base ++ typst_supplement; + return base; +} + fn initSpecs() [specCount()]Spec { var out: [specCount()]Spec = undefined; var i = 0; @@ -55,7 +83,7 @@ fn initSpecs() [specCount()]Spec { .name = g.name, .exts = g.exts, .language = @extern(LanguageFn, .{ .name = "tree_sitter_" ++ g.name }), - .query_src = @field(ts_queries, g.name ++ "_highlights"), + .query_src = querySrc(g), }; i += 1; } @@ -235,6 +263,26 @@ fn synFor(name: []const u8) Syn { .{ "title", .keyword }, .{ "uri", .string }, .{ "reference", .number }, + + // Markup grammars (markdown, typst) name prose constructs in their own + // vocabulary rather than the code vocabulary above, so none of the + // needles so far reach them. These are appended, and first-match-wins + // makes that strictly additive; the needles below were audited across + // all 27 shipped queries and occur only in the markdown and typst ones, + // so no other language is recoloured. + // + // The slot assignment is forced by the palette being four wide and by + // `synStyle` attaching the real BOLD attribute to exactly two of them, + // `keyword` and `comment`: headings take `keyword`, so bold spans have + // to land on `comment` to render actually bold. Nothing is left that + // renders italic, so emphasis can only get a colour shift (`number`). + .{ "heading", .keyword }, + .{ "strong", .comment }, + .{ "bold", .comment }, + .{ "emphasis", .number }, + .{ "italic", .number }, + .{ "literal", .string }, + .{ "raw", .string }, }) |m| { if (std.mem.indexOf(u8, name, m[0]) != null) return m[1]; } @@ -264,19 +312,61 @@ pub fn highlightFileRange(gpa: std.mem.Allocator, path: []const u8, content: []c runQuery(styles, selected, tree, 0); - if (std.mem.eql(u8, selected.name, "markdown")) injectCodeBlocks(styles, source, tree.rootNode()); + if (InjectSite.forGrammar(selected.name)) |site| injectCodeBlocks(styles, source, tree.rootNode(), site); + // Disjoint from the fenced-block pass above: `code_fence_content` is never + // an `inline` node, so the two never write the same byte. + if (std.mem.eql(u8, selected.name, "markdown")) injectMarkdownInline(styles, source, tree.rootNode()); return styles; } -fn injectCodeBlocks(styles: []u8, source: []const u8, node: ts.Node) void { - if (std.mem.eql(u8, node.kind(), "fenced_code_block")) { - highlightCodeBlock(styles, source, node); +// Markdown fenced blocks and Typst raw blocks are the same construct — a +// language tag plus a literal payload — under different node shapes, so one +// walker drives both and only the (lang, content) extraction differs. +const InjectSite = enum { + markdown_fence, + typst_raw, + + fn forGrammar(name: []const u8) ?InjectSite { + if (std.mem.eql(u8, name, "markdown")) return .markdown_fence; + if (std.mem.eql(u8, name, "typst")) return .typst_raw; + return null; + } + + fn blockKind(self: InjectSite) []const u8 { + return switch (self) { + .markdown_fence => "fenced_code_block", + .typst_raw => "raw_blck", + }; + } + + /// null when the block carries no language tag (an untagged fence, or a + /// Typst raw block written without one) — nothing to inject, leave it alone. + fn parts(self: InjectSite, block: ts.Node) ?struct { lang: ts.Node, content: ts.Node } { + switch (self) { + .markdown_fence => { + const info = childOfKind(block, "info_string") orelse return null; + return .{ + .lang = childOfKind(info, "language") orelse return null, + .content = childOfKind(block, "code_fence_content") orelse return null, + }; + }, + .typst_raw => return .{ + .lang = block.childByFieldName("lang") orelse return null, + .content = childOfKind(block, "blob") orelse return null, + }, + } + } +}; + +fn injectCodeBlocks(styles: []u8, source: []const u8, node: ts.Node, site: InjectSite) void { + if (std.mem.eql(u8, node.kind(), site.blockKind())) { + highlightCodeBlock(styles, source, node, site); return; } var i: u32 = 0; const count = node.childCount(); while (i < count) : (i += 1) { - if (node.child(i)) |c| injectCodeBlocks(styles, source, c); + if (node.child(i)) |c| injectCodeBlocks(styles, source, c, site); } } @@ -291,14 +381,12 @@ fn childOfKind(node: ts.Node, kind: []const u8) ?ts.Node { return null; } -fn highlightCodeBlock(styles: []u8, source: []const u8, block: ts.Node) void { - const info = childOfKind(block, "info_string") orelse return; - const lang_node = childOfKind(info, "language") orelse return; - const content_node = childOfKind(block, "code_fence_content") orelse return; - const langtext = source[lang_node.startByte()..lang_node.endByte()]; +fn highlightCodeBlock(styles: []u8, source: []const u8, block: ts.Node, site: InjectSite) void { + const p = site.parts(block) orelse return; + const langtext = source[p.lang.startByte()..p.lang.endByte()]; const sub_sel = (forLang(langtext) catch return) orelse return; - const cs: usize = content_node.startByte(); - const ce: usize = content_node.endByte(); + const cs: usize = p.content.startByte(); + const ce: usize = p.content.endByte(); if (ce > source.len or cs > ce) return; const parser = ts.Parser.create(); @@ -306,9 +394,60 @@ fn highlightCodeBlock(styles: []u8, source: []const u8, block: ts.Node) void { parser.setLanguage(sub_sel.lang) catch return; const tree = parser.parseString(source[cs..ce], null) orelse return; defer tree.destroy(); + // The outer grammar already painted these bytes (Typst blankets the whole + // raw block `string`), and runQuery only writes bytes it captures, so the + // outer colour would survive as a wash behind the injected code. The sub + // grammar owns the payload outright: clear it first. + @memset(styles[cs..ce], @intFromEnum(Syn.none)); runQuery(styles, sub_sel, tree, cs); } +// tree-sitter-markdown is a split grammar: the block parser bottoms out at named +// `inline` nodes whose bytes it never looks inside, and emphasis / +// strong_emphasis / code_span exist only in the companion inline parser. So +// every `inline` node is re-parsed with `markdown_inline`, which is what +// upstream's injections.scm, helix and nvim all do (per node, uncombined — the +// stray block_continuation markers inside a multi-line paragraph's range are +// just plain text to the inline parser). +// +// The grammar is resolved and the parser built once per file, not once per node: +// only the parse is inherently per node. +fn injectMarkdownInline(styles: []u8, source: []const u8, root: ts.Node) void { + // Absent in builds below the `full` tier — nothing to inject, leave the + // block grammar's colours alone. + const sub_sel = (forLang("markdown_inline") catch return) orelse return; + const parser = ts.Parser.create(); + defer parser.destroy(); + parser.setLanguage(sub_sel.lang) catch return; + inlineNodes(styles, source, root, sub_sel, parser); +} + +fn inlineNodes(styles: []u8, source: []const u8, node: ts.Node, sel: Selected, parser: *ts.Parser) void { + if (std.mem.eql(u8, node.kind(), "inline")) { + highlightInline(styles, source, node, sel, parser); + return; // `inline` nodes never nest + } + var i: u32 = 0; + const count = node.childCount(); + while (i < count) : (i += 1) { + if (node.child(i)) |c| inlineNodes(styles, source, c, sel, parser); + } +} + +fn highlightInline(styles: []u8, source: []const u8, node: ts.Node, sel: Selected, parser: *ts.Parser) void { + const s: usize = node.startByte(); + const e: usize = node.endByte(); + if (s >= e or e > source.len) return; // an empty atx heading has a zero-length inline + const tree = parser.parseString(source[s..e], null) orelse return; + defer tree.destroy(); + // Deliberately NOT clearing the range first, unlike highlightCodeBlock: the + // block query already painted a heading's inline text `keyword`, and that is + // the colour the heading must keep wherever the inline pass captures + // nothing. Painting over instead of resetting is what makes *italic* inside + // a heading recolour while the rest of the heading stays heading-coloured. + runQuery(styles, sel, tree, s); +} + /// One Syn byte per byte of content[start, end) for unified diffs/patches. /// Pure byte scan; independent of tree-sitter and the `enabled` flag. Caller frees. pub fn highlightDiff(gpa: std.mem.Allocator, content: []const u8, start_byte_raw: usize, end_byte_raw: usize) ![]u8 { @@ -384,21 +523,104 @@ test "default full grammar set highlights Typst source" { try std.testing.expectEqual(Syn.keyword, @as(Syn, @enumFromInt(short_ext[keyword_at]))); } -test "markdown highlights headings and injects fenced code blocks" { +test "Typst markup constructs paint and raw blocks inject their language" { + if (!enabled or !full_grammars) return; + start(std.testing.allocator); + defer stop(); + + const source = + \\// note + \\= Heading Title + \\Some *bold* text with `raw` inline. + \\#import "mod.typ": helper + \\#let answer = 42 + \\#emit(7, "arg") + \\```zig + \\fn widget() u8 { return 99; } + \\``` + \\ + ; + const styles = try highlightFileRange(std.testing.allocator, "paper.typ", source, 0, source.len); + defer std.testing.allocator.free(styles); + + const synAt = struct { + fn f(s: []const u8, src: []const u8, needle: []const u8, offset: usize) Syn { + return @enumFromInt(s[std.mem.indexOf(u8, src, needle).? + offset]); + } + }.f; + + // marker and text of the heading both take the bold accent + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "= Heading", 0)); + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "Heading", 0)); + // *bold* is @markup.bold, which lands on the one slot that still renders + // with the real bold attribute now that headings own `keyword`. + try std.testing.expectEqual(Syn.comment, synAt(styles, source, "bold", 0)); + try std.testing.expectEqual(Syn.string, synAt(styles, source, "raw` inline", 0)); + // the callee and the code sigil in front of it + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "emit", 0)); + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "#emit", 0)); + + // fence and lang tag keep the literal colour of the raw block... + try std.testing.expectEqual(Syn.string, synAt(styles, source, "```zig", 0)); + try std.testing.expectEqual(Syn.string, synAt(styles, source, "```zig", 3)); + // ...while the blob is reset and re-painted by the injected zig grammar. Its + // `fn` and `99` prove the injection ran; `widget` proves the reset, since the + // zig query names it @function (Syn.none here) and without clearing the blob + // first the raw block's `string` would still be washing over it. + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "fn widget", 0)); + try std.testing.expectEqual(Syn.number, synAt(styles, source, "99", 0)); + try std.testing.expectEqual(Syn.none, synAt(styles, source, "widget", 0)); + + try std.testing.expectEqual(Syn.comment, synAt(styles, source, "// note", 0)); + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "import", 0)); + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "let", 0)); + try std.testing.expectEqual(Syn.number, synAt(styles, source, "42", 0)); + try std.testing.expectEqual(Syn.string, synAt(styles, source, "\"arg\"", 0)); +} + +test "markdown highlights markup, injects inline spans and fenced code blocks" { if (!enabled or !full_grammars) return; start(std.testing.allocator); defer stop(); - const source = "# Heading\n\n```zig\nconst answer = 42;\n```\n"; + const source = + "# Heading *slant* Title\n" ++ + "\n" ++ + "Prose with **stout** and *lean* plus `snippet` inline.\n" ++ + "\n" ++ + "```zig\n" ++ + "fn gadget() u8 { return 77; }\n" ++ + "```\n"; const styles = try highlightFileRange(std.testing.allocator, "doc.md", source, 0, source.len); defer std.testing.allocator.free(styles); - const title_at = std.mem.indexOf(u8, source, "Heading").?; - const num_at = std.mem.indexOf(u8, source, "42").?; - // the heading is markdown's own highlight; the number proves the zig - // grammar was injected into the fenced block and colored its contents. - try std.testing.expectEqual(Syn.keyword, @as(Syn, @enumFromInt(styles[title_at]))); - try std.testing.expectEqual(Syn.number, @as(Syn, @enumFromInt(styles[num_at]))); + const synAt = struct { + fn f(s: []const u8, src: []const u8, needle: []const u8, offset: usize) Syn { + return @enumFromInt(s[std.mem.indexOf(u8, src, needle).? + offset]); + } + }.f; + + // The block grammar washes the whole heading `keyword`; the inline pass then + // paints the emphasis over it without resetting, so the heading keeps its + // colour everywhere the emphasis is not. + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "Title", 0)); + try std.testing.expectEqual(Syn.number, synAt(styles, source, "slant", 0)); + + // bold/italic/code-span live only in the inline grammar, so all three prove + // the inline injection ran. + try std.testing.expectEqual(Syn.comment, synAt(styles, source, "stout", 0)); + try std.testing.expectEqual(Syn.number, synAt(styles, source, "lean", 0)); + try std.testing.expectEqual(Syn.string, synAt(styles, source, "snippet", 0)); + + // the fence is inside the block query's @text.literal wash... + try std.testing.expectEqual(Syn.string, synAt(styles, source, "```zig", 0)); + // ...while the payload is cleared and re-painted by the injected zig + // grammar: `fn` and `77` prove the injection ran, and `gadget` proves the + // clear, since the zig query names it @function (Syn.none here) and without + // clearing first the fence's `string` would still be washing over it. + try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "fn gadget", 0)); + try std.testing.expectEqual(Syn.number, synAt(styles, source, "77", 0)); + try std.testing.expectEqual(Syn.none, synAt(styles, source, "gadget", 0)); } test "highlightDiff colors unified diff lines by prefix" { |