diff options
| author | Gabriel Schneider <[email protected]> | 2026-11-16 22:43:18 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-11-17 12:15:01 -0300 |
| commit | 383b60b79146937e12c82abaaca2ce213d210b7f (patch) | |
| tree | 82fe9805ab85ef72fbc38fa56569632081a8964e | |
| parent | f2b02c455f2bcdfb905a015eefbb630a28b4b12e (diff) | |
| download | pardes-383b60b79146937e12c82abaaca2ce213d210b7f.tar.gz pardes-383b60b79146937e12c82abaaca2ce213d210b7f.zip | |
syntax: the highlight queries are filtered by the build, not in every compile: colorQuery kept each grammar's patterns whose captures pardes colours by scanning the 29 highlights.scm byte by byte at comptime, a quarter of a full recompile of the core (and again in each of the ~15 artifacts that import it); build.zig's treeSitter now runs the same filter once over the query it reads (grammar_manifest.colorQuery, a plain function, with Syn and synFor's capture-to-class table beside it, the one copy both the build and syntax.zig read), so ts_queries' <name>_highlights is the filtered query and Spec.query_src a field read; typst's supplement is the manifest's highlights_extra, appended before the filter as before; <name>_highlights_unfiltered is the whole query, for the test that the filter keeps every colour; a package that depends on pardes gets the same through library's treeSitter, for its own grammar set; proven byte-identical before the comptime path went (29 grammars, 28,398 filtered bytes of 79,678); core-test's forced full recompile 12.7-12.9 s wall / 17.5 s CPU to 9.7-10.1 s / 14.6 s; syntax snapshots unchanged; the changelog says so
| -rw-r--r-- | build.zig | 8 | ||||
| -rw-r--r-- | src/CHANGELOG.md | 6 | ||||
| -rw-r--r-- | src/grammar_manifest.zig | 108 | ||||
| -rw-r--r-- | src/syntax.zig | 114 |
4 files changed, 122 insertions, 114 deletions
@@ -2101,8 +2101,12 @@ fn treeSitter(b: *std.Build, io: std.Io, target: std.Build.ResolvedTarget, c_opt (tree_sitter_grammars == .zig and g.tier == .zig)) grammar: { const dep = b.lazyDependency(g.dep, .{}) orelse break :grammar; - const query = configureRead(b, io, dep.builder.root, g.query, 0x100000) catch @panic("read " ++ g.name ++ " highlights"); - ts_queries.addOption([]const u8, g.name ++ "_highlights", query); + const upstream = configureRead(b, io, dep.builder.root, g.query, 0x100000) catch @panic("read " ++ g.name ++ " highlights"); + // the patterns that colour, filtered here once and not in + // every compile, and the whole query, the filter's test's + const query = std.mem.concat(b.allocator, u8, &.{ upstream, g.highlights_extra }) catch @panic("OOM"); + ts_queries.addOption([]const u8, g.name ++ "_highlights_unfiltered", query); + ts_queries.addOption([]const u8, g.name ++ "_highlights", grammar_manifest.colorQuery(query, b.allocator.alloc(u8, query.len) catch @panic("OOM"))); // helix's textobject queries, vendored (vendor/queries); a // grammar without one gets none const objects = configureRead(b, io, b.root, "vendor/queries/" ++ g.name ++ "/textobjects.scm", 0x100000) catch ""; diff --git a/src/CHANGELOG.md b/src/CHANGELOG.md index dc130be1..3b139f00 100644 --- a/src/CHANGELOG.md +++ b/src/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- An edit to the core recompiles it about 3 s sooner (`core-test`, 13 s to + 10 s): the build keeps each grammar's highlight patterns that pardes + colours once, where every compile of every program that takes the core + did it again, a quarter of its time. A package that depends on pardes + is given the same filtered queries. + - Commit patches open as `+Diff-<short-hash>` and reuse only their own commit's pane, so another paged patch no longer replaces one already open. diff --git a/src/grammar_manifest.zig b/src/grammar_manifest.zig index 38c54916..711c31a4 100644 --- a/src/grammar_manifest.zig +++ b/src/grammar_manifest.zig @@ -1,6 +1,7 @@ //! Dependency-free tree-sitter grammar metadata shared by build.zig and the -//! runtime syntax registry. Keep target symbols and query values out of this -//! file: the build runner imports it as host code. +//! runtime syntax registry, and the highlight-query filter the build runs. +//! Keep target symbols and query values out of this file: the build runner +//! imports it as host code. const std = @import("std"); @@ -14,6 +15,8 @@ pub const Grammar = struct { src: []const u8 = "src", scanner: bool = false, query: []const u8 = "queries/highlights.scm", + /// Patterns appended to the highlights query before it is filtered. + highlights_extra: []const u8 = "", }; pub const all = [_]Grammar{ @@ -48,7 +51,13 @@ pub const all = [_]Grammar{ .{ .name = "ruby", .dep = "ts_ruby", .exts = &.{ ".rb", ".rake" }, .tier = .full, .scanner = true }, .{ .name = "rust", .dep = "ts_rust", .exts = &.{".rs"}, .tier = .full, .scanner = true }, .{ .name = "scala", .dep = "ts_scala", .exts = &.{ ".scala", ".sc" }, .tier = .full, .scanner = true }, - .{ .name = "typst", .dep = "ts_typst", .exts = &.{ ".typ", ".typst" }, .tier = .full, .scanner = true, .query = "queries/typst/highlights.scm" }, + .{ .name = "typst", .dep = "ts_typst", .exts = &.{ ".typ", ".typst" }, .tier = .full, .scanner = true, .query = "queries/typst/highlights.scm", .highlights_extra = + \\ + \\(call item: (ident) @keyword) + \\(call item: (field field: (ident) @keyword)) + \\"#" @keyword + \\ + }, .{ .name = "zig", .dep = "ts_zig", .exts = &.{ ".zig", ".zon" }, .tier = .zig }, }; @@ -92,3 +101,96 @@ pub fn byPath(path: []const u8) ?usize { for (all, 0..) |g, i| for (g.exts) |e| if (std.ascii.eqlIgnoreCase(ext, e)) return i; return null; } + +/// pardes's syntax classes: what a capture colours a byte as. +pub const Syn = enum(u8) { none, keyword, string, number, comment }; + +pub fn synFor(name: []const u8) Syn { + for ([_]struct { []const u8, Syn }{ + .{ "comment", .comment }, + .{ "string", .string }, + .{ "character", .string }, + .{ "number", .number }, + .{ "numeric", .number }, + .{ "float", .number }, + .{ "boolean", .number }, + .{ "keyword", .keyword }, + .{ "include", .keyword }, + .{ "conditional", .keyword }, + .{ "repeat", .keyword }, + .{ "title", .keyword }, + .{ "uri", .string }, + .{ "reference", .number }, + .{ "heading", .keyword }, + .{ "strong", .comment }, + .{ "bold", .comment }, + .{ "emphasis", .number }, + .{ "italic", .number }, + .{ "literal", .string }, + .{ "raw", .string }, + }) |m| { + if (std.mem.indexOf(u8, name, m[0]) != null) return m[1]; + } + return .none; +} + +/// The patterns of highlights query `source` that capture a class (synFor), +/// written to `out` (source.len bytes): what a grammar's query compiles to. +/// build.zig filters each one with it, ts_queries' `<name>_highlights`. +pub fn colorQuery(source: []const u8, out: []u8) []u8 { + var written: usize = 0; + var at: usize = 0; + while (at < source.len) { + while (at < source.len) { + if (std.ascii.isWhitespace(source[at])) { + at += 1; + } else if (source[at] == ';') { + while (at < source.len and source[at] != '\n') at += 1; + } else break; + } + const begin = at; + var depth: usize = 0; + var colored = false; + var complete = false; + while (at < source.len) { + const byte = source[at]; + if (complete and (byte == '(' or byte == '[' or byte == '"')) break; + if (byte == ';') { + while (at < source.len and source[at] != '\n') at += 1; + continue; + } + if (byte == '"') { + at += 1; + while (at < source.len) : (at += 1) { + if (source[at] == '\\') { + at += 1; + } else if (source[at] == '"') { + at += 1; + break; + } + } + if (depth == 0) complete = true; + continue; + } + if (byte == '(' or byte == '[') depth += 1; + if (byte == ')' or byte == ']') { + depth -= 1; + if (depth == 0) complete = true; + } + if (byte == '@') { + const name = at + 1; + at = name; + while (at < source.len and (std.ascii.isAlphanumeric(source[at]) or + source[at] == '_' or source[at] == '.' or source[at] == '-')) at += 1; + colored = colored or synFor(source[name..at]) != .none; + continue; + } + at += 1; + } + if (colored) { + @memcpy(out[written..][0 .. at - begin], source[begin..at]); + written += at - begin; + } + } + return out[0..written]; +} diff --git a/src/syntax.zig b/src/syntax.zig index 2526e113..8533aaa9 100644 --- a/src/syntax.zig +++ b/src/syntax.zig @@ -22,7 +22,7 @@ const ts_queries = if (enabled) @import("ts_queries") else struct {}; const LanguageFn = *const fn () callconv(.c) *const ts.Language; -pub const Syn = enum(u8) { none, keyword, string, number, comment }; +pub const Syn = grammar_manifest.Syn; /// A query compiled when first asked for: "" for none, null when it fails. const Lazy = struct { src: []const u8 = "", query: ?*ts.Query = null, failed: bool = false }; @@ -55,81 +55,6 @@ fn specCount() comptime_int { } return count; } -const typst_supplement = - \\ - \\(call item: (ident) @keyword) - \\(call item: (field field: (ident) @keyword)) - \\"#" @keyword - \\ -; - -fn querySrc(comptime g: grammar_manifest.Grammar) []const u8 { - const base = @field(ts_queries, g.name ++ "_highlights"); - const source = if (comptime std.mem.eql(u8, g.name, "typst")) base ++ typst_supplement else base; - return comptime colorQuery(source); -} - -fn colorQuery(comptime source: []const u8) []const u8 { - @setEvalBranchQuota(2_000_000); - var result: [source.len]u8 = undefined; - var written: usize = 0; - var at: usize = 0; - while (at < source.len) { - while (at < source.len) { - if (std.ascii.isWhitespace(source[at])) { - at += 1; - } else if (source[at] == ';') { - while (at < source.len and source[at] != '\n') at += 1; - } else break; - } - const begin = at; - var depth: usize = 0; - var colored = false; - var complete = false; - while (at < source.len) { - const byte = source[at]; - if (complete and (byte == '(' or byte == '[' or byte == '"')) break; - if (byte == ';') { - while (at < source.len and source[at] != '\n') at += 1; - continue; - } - if (byte == '"') { - at += 1; - while (at < source.len) : (at += 1) { - if (source[at] == '\\') { - at += 1; - } else if (source[at] == '"') { - at += 1; - break; - } - } - if (depth == 0) complete = true; - continue; - } - if (byte == '(' or byte == '[') depth += 1; - if (byte == ')' or byte == ']') { - depth -= 1; - if (depth == 0) complete = true; - } - if (byte == '@') { - const name = at + 1; - at = name; - while (at < source.len and (std.ascii.isAlphanumeric(source[at]) or - source[at] == '_' or source[at] == '.' or source[at] == '-')) at += 1; - colored = colored or synFor(source[name..at]) != .none; - continue; - } - at += 1; - } - if (colored) { - @memcpy(result[written..][0 .. at - begin], source[begin..at]); - written += at - begin; - } - } - const filtered = result[0..written].*; - return &filtered; -} - fn initSpecs() [specCount()]Spec { var out: [specCount()]Spec = undefined; var i = 0; @@ -139,7 +64,7 @@ fn initSpecs() [specCount()]Spec { .name = g.name, .exts = g.exts, .language = @extern(LanguageFn, .{ .name = "tree_sitter_" ++ g.name }), - .query_src = querySrc(g), + .query_src = @field(ts_queries, g.name ++ "_highlights"), .objects = .{ .src = @field(ts_queries, g.name ++ "_textobjects") }, .indents = .{ .src = @field(ts_queries, g.name ++ "_indents") }, }; @@ -219,7 +144,7 @@ fn ensure(spec: *Spec) !Selected { const capture_styles = spec.capture_styles[0..query.captureCount()]; for (capture_styles, 0..) |*style, id| { const name = query.captureNameForId(@intCast(id)) orelse ""; - style.* = @backingInt(synFor(name)); + style.* = @backingInt(grammar_manifest.synFor(name)); if (style.* == 0) query.disableCapture(name); } const parser = ts.Parser.create(); @@ -260,34 +185,6 @@ fn runQuery(styles: []u8, sel: Selected, tree: *ts.Tree, base: usize) void { } } -fn synFor(name: []const u8) Syn { - for ([_]struct { []const u8, Syn }{ - .{ "comment", .comment }, - .{ "string", .string }, - .{ "character", .string }, - .{ "number", .number }, - .{ "numeric", .number }, - .{ "float", .number }, - .{ "boolean", .number }, - .{ "keyword", .keyword }, - .{ "include", .keyword }, - .{ "conditional", .keyword }, - .{ "repeat", .keyword }, - .{ "title", .keyword }, - .{ "uri", .string }, - .{ "reference", .number }, - .{ "heading", .keyword }, - .{ "strong", .comment }, - .{ "bold", .comment }, - .{ "emphasis", .number }, - .{ "italic", .number }, - .{ "literal", .string }, - .{ "raw", .string }, - }) |m| { - if (std.mem.indexOf(u8, name, m[0]) != null) return m[1]; - } - return .none; -} pub fn highlightFileRange(gpa: std.mem.Allocator, path: []const u8, content: []const u8, start_raw: usize, end_raw: usize) ![]u8 { const zone = tracy.zone(@src(), "highlightFileRange"); defer zone.end(); @@ -1909,8 +1806,7 @@ test "syntax query filtering preserves upstream colors" { inline for (grammar_manifest.all) |grammar| { if (comptime grammarSelected(grammar)) { const selected = (try forLang(grammar.name)).?; - const raw_source = @field(ts_queries, grammar.name ++ "_highlights") ++ - (if (comptime std.mem.eql(u8, grammar.name, "typst")) typst_supplement else ""); + const raw_source = @field(ts_queries, grammar.name ++ "_highlights_unfiltered"); var error_offset: u32 = 0; const raw_query = try ts.Query.create(selected.lang, raw_source, &error_offset); defer raw_query.destroy(); @@ -1918,7 +1814,7 @@ test "syntax query filtering preserves upstream colors" { reference.query = raw_query; reference.capture_styles = try std.testing.allocator.alloc(u8, raw_query.captureCount()); defer std.testing.allocator.free(reference.capture_styles); - for (reference.capture_styles, 0..) |*style, id| style.* = @backingInt(synFor(raw_query.captureNameForId(@intCast(id)) orelse "")); + for (reference.capture_styles, 0..) |*style, id| style.* = @backingInt(grammar_manifest.synFor(raw_query.captureNameForId(@intCast(id)) orelse "")); const tree = selected.parser.parseString(source, null) orelse return error.ParseFailed; defer tree.destroy(); var expected: [source.len]u8 = @splat(0); |
