diff options
| author | Gabriel Schneider <[email protected]> | 2026-10-30 17:35:06 -0300 |
|---|---|---|
| committer | Gabriel Schneider <[email protected]> | 2026-10-30 17:35:06 -0300 |
| commit | 6d8da8721141284cf33bb901168b26135893eb86 (patch) | |
| tree | 791d63401d4e85fbca1d597a8ce0326e70efd545 /src/syntax.zig | |
| parent | efd43a3a2cf03bb741366331158b368abf9ac583 (diff) | |
| parent | bf6810a8f9d6cc82050e7d9c0cfb06358301b574 (diff) | |
| download | pardes-6d8da8721141284cf33bb901168b26135893eb86.tar.gz pardes-6d8da8721141284cf33bb901168b26135893eb86.zip | |
Merge latest upstream into the macOS branch for Zig 0.17
Retain the macOS branch history and local Acme selection color while adopting upstream's current core and platform changes. Pin Zig and ZLS to 0.17.0, and fetch the existing zstbi compatibility commit through upstream after its original fork was removed.
Diffstat (limited to 'src/syntax.zig')
| -rw-r--r-- | src/syntax.zig | 1090 |
1 files changed, 919 insertions, 171 deletions
diff --git a/src/syntax.zig b/src/syntax.zig index e637bf67..985a9888 100644 --- a/src/syntax.zig +++ b/src/syntax.zig @@ -3,6 +3,13 @@ const config = @import("pardes_config"); const tracy = @import("tracy.zig"); const grammar_manifest = @import("grammar_manifest.zig"); const look = @import("look.zig"); +const regexp = @import("regexp.zig"); +const c_heap = @import("c_heap"); +const diff = @import("diff.zig"); + +test { + _ = diff; +} pub const enabled = config.syntax_highlighting; const zig_grammar = config.syntax_zig_grammar; @@ -26,6 +33,10 @@ const Spec = struct { exts: []const []const u8, language: *const fn () callconv(.c) *const ts.Language, query_src: []const u8, + /// helix's textobjects.scm for the grammar (vendor/queries), "" for none + objects_src: []const u8 = "", + objects: ?*ts.Query = null, + objects_failed: bool = false, selected: ?Selected = null, capture_styles: [256]u8 = undefined, }; @@ -130,6 +141,7 @@ fn initSpecs() [specCount()]Spec { .exts = g.exts, .language = @extern(LanguageFn, .{ .name = "tree_sitter_" ++ g.name }), .query_src = querySrc(g), + .objects_src = @field(ts_queries, g.name ++ "_textobjects"), }; i += 1; } @@ -139,8 +151,11 @@ fn initSpecs() [specCount()]Spec { var specs = initSpecs(); -const allocation_header_size = 16; +/// What tree-sitter allocates from, the runtime and every grammar's external +/// scanner (build.zig compiles them to use the runtime's allocator): +/// `start`'s allocator, through the shared C heap. var syntax_allocator: std.mem.Allocator = undefined; +const syntax_heap = c_heap.Heap(&syntax_allocator); var syntax_started = false; extern fn ts_set_allocator( @@ -150,56 +165,13 @@ extern fn ts_set_allocator( new_free: ?*const fn (ptr: ?*anyopaque) callconv(.c) void, ) void; -fn syntaxAlloc(size_arg: usize) callconv(.c) ?*anyopaque { - const size = @max(size_arg, 1); - const total = std.math.add(usize, allocation_header_size, size) catch return null; - const bytes = syntax_allocator.alignedAlloc(u8, .@"16", total) catch return null; - const header: *align(16) usize = @ptrCast(bytes.ptr); - header.* = total; - return @ptrCast(bytes.ptr + allocation_header_size); -} - -fn syntaxCalloc(count: usize, size: usize) callconv(.c) ?*anyopaque { - const len = std.math.mul(usize, count, size) catch return null; - const pointer = syntaxAlloc(len) orelse return null; - const bytes: [*]u8 = @ptrCast(pointer); - @memset(bytes[0..len], 0); - return pointer; -} - -fn syntaxRealloc(ptr: ?*anyopaque, new_size: usize) callconv(.c) ?*anyopaque { - const pointer = ptr orelse return syntaxAlloc(new_size); - if (new_size == 0) { - syntaxFree(pointer); - return null; - } - const user: [*]u8 = @ptrCast(pointer); - const header: *align(16) usize = @ptrCast(@alignCast(user - allocation_header_size)); - const old_total = header.*; - const old_bytes: []align(16) u8 = @as([*]align(16) u8, @ptrCast(header))[0..old_total]; - const new_total = std.math.add(usize, allocation_header_size, new_size) catch return null; - const new_bytes = syntax_allocator.realloc(old_bytes, new_total) catch return null; - const new_header: *align(16) usize = @ptrCast(new_bytes.ptr); - new_header.* = new_total; - return @ptrCast(new_bytes.ptr + allocation_header_size); -} - -fn syntaxFree(ptr: ?*anyopaque) callconv(.c) void { - const pointer = ptr orelse return; - const user: [*]u8 = @ptrCast(pointer); - const header: *align(16) usize = @ptrCast(@alignCast(user - allocation_header_size)); - const total = header.*; - const bytes: []align(16) u8 = @as([*]align(16) u8, @ptrCast(header))[0..total]; - syntax_allocator.free(bytes); -} - pub fn start(gpa: std.mem.Allocator) void { if (comptime enabled) { std.debug.assert(!syntax_started); stop(); syntax_allocator = gpa; syntax_started = true; - ts_set_allocator(syntaxAlloc, syntaxCalloc, syntaxRealloc, syntaxFree); + ts_set_allocator(syntax_heap.malloc, syntax_heap.calloc, syntax_heap.realloc, syntax_heap.free); } } @@ -212,7 +184,11 @@ pub fn stop() void { selected.query.destroy(); } spec.selected = null; + if (spec.objects) |q| q.destroy(); + spec.objects = null; + spec.objects_failed = false; } + dropDiffPieces(); ts_set_allocator(null, null, null, null); syntax_started = false; syntax_allocator = undefined; @@ -238,7 +214,7 @@ fn ensure(spec: *Spec) !Selected { const capture_styles = spec.capture_styles[0..query.captureCount()]; for (capture_styles, 0..) |*style, id| { const name = query.captureNameForId(@intCast(id)) orelse ""; - style.* = @intFromEnum(synFor(name)); + style.* = @backingInt(synFor(name)); if (style.* == 0) query.disableCapture(name); } const parser = ts.Parser.create(); @@ -265,35 +241,8 @@ fn forExt(ext: []const u8) !?Selected { return null; } -const LangAlias = struct { []const u8, []const u8 }; -const lang_aliases = [_]LangAlias{ - .{ "js", "javascript" }, - .{ "jsx", "javascript" }, - .{ "mjs", "javascript" }, - .{ "py", "python" }, - .{ "sh", "bash" }, - .{ "shell", "bash" }, - .{ "zsh", "bash" }, - .{ "bash", "bash" }, - .{ "rs", "rust" }, - .{ "c++", "cpp" }, - .{ "cxx", "cpp" }, - .{ "cc", "cpp" }, - .{ "cs", "c_sharp" }, - .{ "kt", "kotlin" }, - .{ "rb", "ruby" }, - .{ "ml", "ocaml" }, - .{ "hs", "haskell" }, -}; - fn forLang(name: []const u8) !?Selected { - var canonical = name; - for (lang_aliases) |a| { - if (std.ascii.eqlIgnoreCase(name, a[0])) { - canonical = a[1]; - break; - } - } + const canonical = if (grammar_manifest.byName(name)) |i| grammar_manifest.all[i].name else name; for (&specs) |*spec| { if (std.ascii.eqlIgnoreCase(canonical, spec.name)) return try ensure(spec); } @@ -355,6 +304,56 @@ pub fn highlightFileRange(gpa: std.mem.Allocator, path: []const u8, content: []c return styles; } +/// Whether `highlightApart` can colour `path` on a worker: a grammar with +/// no injections (markdown's and typst's reach the shared parsers), made +/// ready here, on the loop, so the worker only reads its query. +pub fn readyApart(path: []const u8) bool { + if (!enabled) return false; + const selected = (forExt(std.fs.path.extension(path)) catch return false) orelse return false; + return !std.mem.eql(u8, selected.name, "markdown") and !std.mem.eql(u8, selected.name, "typst"); +} + +/// `highlightFileRange` of `source` (a copy of the range), on a worker: a +/// parser and a query cursor of its own, the grammar's query shared (it +/// is only read); `halt` ends the parse early. After `readyApart`. +pub fn highlightApart(gpa: std.mem.Allocator, path: []const u8, source: []const u8, halt: *const std.atomic.Value(bool)) ![]u8 { + if (comptime !enabled) return &.{}; + const shared = (try forExt(std.fs.path.extension(path))) orelse return &.{}; + const parser = ts.Parser.create(); + defer parser.destroy(); + try parser.setLanguage(shared.lang); + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + const Reader = struct { + text: []const u8, + fn read(payload: ?*anyopaque, byte: u32, _: ts.Point, n: *u32) callconv(.c) [*c]const u8 { + const r: *@This() = @ptrCast(@alignCast(payload.?)); + const at = @min(byte, r.text.len); + n.* = @intCast(r.text.len - at); + return r.text.ptr + at; + } + fn progress(state: *ts.Parser.State) callconv(.c) bool { + const flag: *const std.atomic.Value(bool) = @ptrCast(@alignCast(state.payload.?)); + return flag.load(.monotonic); + } + }; + var reader: Reader = .{ .text = source }; + const tree = parser.parseWithOptions( + .{ .payload = &reader, .read = Reader.read, .encoding = .utf8 }, + null, + .{ .payload = @constCast(halt), .progress_callback = @ptrCast(&Reader.progress) }, + ) orelse return error.Canceled; + defer tree.destroy(); + if (halt.load(.monotonic)) return error.Canceled; + const styles = try gpa.alloc(u8, source.len); + @memset(styles, 0); + var own = shared; + own.parser = parser; + own.cursor = cursor; + runQuery(styles, own, tree, 0); + return styles; +} + /// Source spans are independent of editor rows. Locations output uses the same /// declarations as sticky editor headers. Byte ends are exclusive; rows are /// zero-based and inclusive, including the last signature/opening-brace row. @@ -376,6 +375,233 @@ pub fn supportsPath(path: []const u8) bool { return false; } +/// A whole file's parse, kept by the file for the keys that walk its nodes +/// (helix's Alt-o and kin). Opaque outside this file; null without a grammar. +pub fn parseTree(path: []const u8, content: []const u8) ?*anyopaque { + if (!enabled) return null; + const selected = (forExt(std.fs.path.extension(path)) catch return null) orelse return null; + const tree = selected.parser.parseString(content, null) orelse return null; + return @ptrCast(tree); +} + +pub fn freeTree(tree: *anyopaque) void { + if (!enabled) return; + const t: *ts.Tree = @ptrCast(@alignCast(tree)); + t.destroy(); +} + +/// Where a node walk goes from a range (helix object.rs and movement.rs). +pub const NodeWalk = @import("modal.zig").Normal.NodeWalk; +pub const Span = struct { from: usize, to: usize }; + +/// The spans `walk` takes the byte range [from, to) to, into `out`; `at` is +/// the range's cursor. None when the tree has nothing to say there. +pub fn walkNodes(tree_ptr: *anyopaque, walk: NodeWalk, from: usize, to: usize, at: usize, out: []Span) usize { + if (!enabled or out.len == 0) return 0; + const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr)); + const lo: u32 = @intCast(@min(from, std.math.maxInt(u32))); + const hi: u32 = @intCast(@min(to, std.math.maxInt(u32))); + if (walk == .parent_end or walk == .parent_start) { + var node = tree.rootNode().namedDescendantForByteRange(lo, hi) orelse return 0; + if (walk == .parent_end) { + out[0] = .{ .from = node.endByte(), .to = node.endByte() }; + return 1; + } + // already at the node's lo: its first ancestor that starts earlier + if (node.startByte() == at) { + const first = node.startByte(); + while (node.startByte() >= first or !node.isNamed()) node = node.parent() orelse break; + } + out[0] = .{ .from = node.startByte(), .to = node.startByte() }; + return 1; + } + var cursor = tree.walk(); + defer cursor.destroy(); + // helix TreeCursor.reset_to_byte_range: the smallest node holding the range + while (true) { + const node = cursor.node(); + if (lo < node.startByte() or hi > node.endByte()) { + _ = cursor.gotoParent(); + break; + } + if (cursor.gotoFirstChildForByte(lo) == null) break; + } + switch (walk) { + .expand => while (cursor.node().startByte() == lo and cursor.node().endByte() == hi) { + if (!cursor.gotoParent()) break; + }, + .shrink => _ = cursor.gotoFirstChild(), + .next_sibling => while (!cursor.gotoNextSibling()) { + if (!cursor.gotoParent()) break; + }, + .prev_sibling => while (!cursor.gotoPreviousSibling()) { + if (!cursor.gotoParent()) break; + }, + .all_siblings, .all_children => { + if (walk == .all_siblings) { + while (true) { + if (!cursor.gotoParent()) return 0; + if (cursor.node().childCount() > 1) break; + } + } + // the named children, or nothing to say + var n: usize = 0; + if (!cursor.gotoFirstChild()) return 0; + while (true) { + const child = cursor.node(); + if (child.isNamed() and n < out.len) { + out[n] = .{ .from = child.startByte(), .to = child.endByte() }; + n += 1; + } + if (!cursor.gotoNextSibling()) break; + } + return n; + }, + .parent_end, .parent_start => unreachable, + } + const node = cursor.node(); + out[0] = .{ .from = node.startByte(), .to = node.endByte() }; + return 1; +} + +/// helix's textobjects, which its `]f`, `mif` and kin select: the name a +/// query's captures begin with (`function.around`, `test.inside`). +pub const Object = @import("modal.zig").Normal.Object; + +fn objectName(o: Object) []const u8 { + return if (o == .xml_element) "xml-element" else @tagName(o); +} + +/// The grammar's textobject query for `path`, compiled the first time; null +/// when the grammar has none. +fn objectQuery(path: []const u8) ?*ts.Query { + const ext = std.fs.path.extension(path); + for (&specs) |*spec| for (spec.exts) |choice| { + if (!std.ascii.eqlIgnoreCase(ext, choice)) continue; + if (spec.objects) |q| return q; + if (spec.objects_src.len == 0 or spec.objects_failed) return null; + var error_offset: u32 = 0; + spec.objects = ts.Query.create(spec.language(), spec.objects_src, &error_offset) catch { + spec.objects_failed = true; + return null; + }; + return spec.objects; + }; + return null; +} + +fn captureId(q: *const ts.Query, name: []const u8) ?u32 { + for (0..q.captureCount()) |i| if (std.mem.eql(u8, q.captureNameForId(@intCast(i)) orelse "", name)) return @intCast(i); + return null; +} + +/// A match's `capture`, as one span over all its nodes (helix's grouped +/// capture), when the match's `#eq?` and `#match?` hold. +fn matchSpan(q: *const ts.Query, match: anytype, capture: u32, text: []const u8) ?Span { + var span: ?Span = null; + for (match.captures) |c| if (c.index == capture) { + const s: Span = .{ .from = c.node.startByte(), .to = c.node.endByte() }; + span = if (span) |old| .{ .from = @min(old.from, s.from), .to = @max(old.to, s.to) } else s; + }; + var found = span orelse return null; + // a node that takes its line's newline (this zig grammar's comments) + // is the line without it, as helix's grammars have it + if (found.to > found.from + 1 and found.to <= text.len and text[found.to - 1] == '\n') found.to -= 1; + const steps = q.predicatesForPattern(match.pattern_index); + var i: usize = 0; + while (i < steps.len) { + var end = i; + while (end < steps.len and steps[end].type != .done) end += 1; + const pred = steps[i..end]; + i = end + 1; + if (pred.len != 3 or pred[0].type != .string or pred[1].type != .capture) continue; + const op = q.stringValueForId(pred[0].value_id) orelse continue; + const subject = for (match.captures) |c| { + if (c.index == pred[1].value_id) break text[@min(c.node.startByte(), text.len)..@min(c.node.endByte(), text.len)]; + } else continue; + const want = if (pred[2].type == .string) q.stringValueForId(pred[2].value_id) orelse continue else for (match.captures) |c| { + if (c.index == pred[2].value_id) break text[@min(c.node.startByte(), text.len)..@min(c.node.endByte(), text.len)]; + } else continue; + if (std.mem.eql(u8, op, "eq?")) { + if (!std.mem.eql(u8, subject, want)) return null; + } else if (std.mem.eql(u8, op, "match?")) { + var re = regexp.Regex.compile(want) catch continue; + if ((re.find(subject, 0, subject.len, subject.len) catch null) == null) return null; + } + } + return found; +} + +/// `mi`/`ma`: the smallest `object.inside` (or `.around`) holding byte `at` +/// (helix textobject_treesitter). None without a query or such an object. +pub fn objectAt(path: []const u8, tree_ptr: *anyopaque, text: []const u8, object: Object, around: bool, at: usize) ?Span { + if (!enabled) return null; + const q = objectQuery(path) orelse return null; + var buf: [32]u8 = undefined; + const want = std.fmt.bufPrint(&buf, "{s}.{s}", .{ objectName(object), if (around) "around" else "inside" }) catch return null; + const capture = captureId(q, want) orelse return null; + const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr)); + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + cursor.exec(q, tree.rootNode()); + var best: ?Span = null; + while (cursor.nextMatch()) |match| { + const s = matchSpan(q, match, capture, text) orelse continue; + if (!(s.from <= at and at < s.to)) continue; + if (best == null or s.to - s.from < best.?.to - best.?.from) best = s; + } + const s = best orelse return null; + if (s.from >= text.len or s.to >= text.len) return null; + return s; +} + +/// `]f`/`[f` and kin: the next object starting after byte `at`, or the +/// previous one ending before it (helix goto_treesitter_object), from the +/// first of `.movement`, `.around`, `.inside` the query names. +pub fn objectNext(path: []const u8, tree_ptr: *anyopaque, text: []const u8, object: Object, forward: bool, at: usize) ?Span { + if (!enabled) return null; + const q = objectQuery(path) orelse return null; + var buf: [32]u8 = undefined; + const capture = for ([_][]const u8{ "movement", "around", "inside" }) |kind| { + const want = std.fmt.bufPrint(&buf, "{s}.{s}", .{ objectName(object), kind }) catch return null; + if (captureId(q, want)) |id| break id; + } else return null; + const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr)); + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + cursor.exec(q, tree.rootNode()); + var best: ?Span = null; + while (cursor.nextMatch()) |match| { + const s = matchSpan(q, match, capture, text) orelse continue; + if (forward) { + if (s.from <= at) continue; + if (best == null or s.from < best.?.from or (s.from == best.?.from and s.to > best.?.to)) best = s; + } else { + if (s.to >= at) continue; + if (best == null or s.to > best.?.to or (s.to == best.?.to and s.from < best.?.from)) best = s; + } + } + const s = best orelse return null; + if (s.from >= text.len or s.to >= text.len) return null; + return s; +} + +test "every vendored textobject query compiles against its grammar" { + if (!enabled) return; + var failed: usize = 0; + for (&specs) |*spec| { + if (spec.objects_src.len == 0) continue; + var error_offset: u32 = 0; + const q = ts.Query.create(spec.language(), spec.objects_src, &error_offset) catch { + std.debug.print("textobjects for {s} do not compile (offset {d})\n", .{ spec.name, error_offset }); + failed += 1; + continue; + }; + q.destroy(); + } + try std.testing.expectEqual(@as(usize, 0), failed); +} + /// Owned source analysis. Both slices use the allocator passed to analyzeSource. /// The parsed tree is released before returning; callers can cache this result. pub const SourceAnalysis = struct { @@ -396,7 +622,7 @@ pub fn analyzeSource(gpa: std.mem.Allocator, path: []const u8, content: []const defer tree.destroy(); var result: SourceAnalysis = .{}; errdefer result.deinit(gpa); - if (want_declarations) result.declarations = try declarationsFromTree(gpa, content, tree); + if (want_declarations) result.declarations = try declarationsFromTree(gpa, content, tree, null); if (want_colors) { result.colors = try gpa.alloc(u8, content.len); @memset(result.colors, 0); @@ -410,14 +636,55 @@ pub fn contextDeclarations(gpa: std.mem.Allocator, path: []const u8, content: [] return analysis.declarations; } -fn declarationsFromTree(gpa: std.mem.Allocator, content: []const u8, tree: *ts.Tree) ![]ContextDeclaration { +/// The declarations of `content` (its path's language), parsed with a +/// parser of its own, so a worker thread may ask: the loop's parsers +/// (`ensure`'s, one a language) are not touched, and a language is safe to +/// share. `stop`, set from another thread, ends the parse and the walk +/// early (error.Canceled): a 1.2 MB file is seconds in a Debug build. +pub fn declarationsApart(gpa: std.mem.Allocator, path: []const u8, content: []const u8, halt: *const std.atomic.Value(bool)) ![]ContextDeclaration { + if (comptime !enabled) return &.{}; + if (content.len == 0) return &.{}; + const ext = std.fs.path.extension(path); + const spec = specs: for (&specs) |*spec| { + for (spec.exts) |choice| if (std.ascii.eqlIgnoreCase(ext, choice)) break :specs spec; + } else return &.{}; + const parser = ts.Parser.create(); + defer parser.destroy(); + try parser.setLanguage(spec.language()); + const Reader = struct { + text: []const u8, + fn read(payload: ?*anyopaque, byte: u32, _: ts.Point, n: *u32) callconv(.c) [*c]const u8 { + const r: *@This() = @ptrCast(@alignCast(payload.?)); + const at = @min(byte, r.text.len); + n.* = @intCast(r.text.len - at); + return r.text.ptr + at; + } + fn progress(state: *ts.Parser.State) callconv(.c) bool { + const flag: *const std.atomic.Value(bool) = @ptrCast(@alignCast(state.payload.?)); + return flag.load(.monotonic); + } + }; + var reader: Reader = .{ .text = content }; + const tree = parser.parseWithOptions( + .{ .payload = &reader, .read = Reader.read, .encoding = .utf8 }, + null, + .{ .payload = @constCast(halt), .progress_callback = @ptrCast(&Reader.progress) }, + ) orelse return error.Canceled; + defer tree.destroy(); + return declarationsFromTree(gpa, content, tree, halt); +} + +fn declarationsFromTree(gpa: std.mem.Allocator, content: []const u8, tree: *ts.Tree, halt: ?*const std.atomic.Value(bool)) ![]ContextDeclaration { var result: std.ArrayList(ContextDeclaration) = .empty; errdefer result.deinit(gpa); // The cursor retains its ancestor stack: sibling visits do not repeatedly // reconstruct parents in broad syntax trees. Preorder preserves source order. var cursor = tree.rootNode().walk(); defer cursor.destroy(); + var visited: usize = 0; while (true) { + visited += 1; + if (halt) |flag| if (visited % 4096 == 0 and flag.load(.monotonic)) return error.Canceled; const node = cursor.node(); if (node.isNamed()) if (contextSpan(node, content)) |span| { if (result.items.len == 0 or result.items[result.items.len - 1].start_line != span.start_line or @@ -677,7 +944,13 @@ pub fn highlightLocationRows(gpa: std.mem.Allocator, content: []const u8, rows: if (comptime @hasField(@TypeOf(row), "location_end")) { address_only = row.location_end > 0 and source_start == end; } - if (!address_only) { + // A row its producer coloured whole is coloured below: no + // fragment of it to parse. + var coloured = false; + if (comptime @hasField(@TypeOf(row), "colors")) { + coloured = row.colors.len > 0 and row.colors.len >= end - source_start; + } + if (!address_only and !coloured) { sources[source_count] = .{ .index = index, .start = source_start, .len = end - source_start }; source_count += 1; } @@ -757,13 +1030,10 @@ pub fn highlightLocationRows(gpa: std.mem.Allocator, content: []const u8, rows: } fn codeAfterLocation(line: []const u8) ?struct { path: []const u8, at: usize, text: []const u8 } { - const token_end = std.mem.indexOfAny(u8, line, " \t") orelse return null; - if (token_end == 0) return null; - const token = line[0..token_end]; - const target = look.parsePathLine(token); - if (target.end != token.len) return null; + const trimmed = std.mem.trimStart(u8, line, " \t"); + const target = @import("locations.zig").parseRaw(trimmed); if (target.at.line == 0) return null; - const at = token_end + 1; + const at = line.len - trimmed.len + target.code_start; if (at >= line.len) return null; return .{ .path = target.path, .at = at, .text = line[at..] }; } @@ -786,11 +1056,11 @@ test "syntax formatted locations color hidden context and preserve multiline sou const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); const body = std.mem.indexOf(u8, content, "comment body").?; - try std.testing.expectEqual(@intFromEnum(Syn.comment), styles[body]); - try std.testing.expectEqual(@intFromEnum(Syn.none), styles[body - 1]); + try std.testing.expectEqual(@backingInt(Syn.comment), styles[body]); + try std.testing.expectEqual(@backingInt(Syn.none), styles[body - 1]); const number = std.mem.indexOf(u8, content, "42").?; - try std.testing.expectEqual(@intFromEnum(Syn.number), styles[number]); - try std.testing.expectEqual(@intFromEnum(Syn.none), styles[0]); + try std.testing.expectEqual(@backingInt(Syn.number), styles[number]); + try std.testing.expectEqual(@backingInt(Syn.none), styles[0]); } test "syntax formatted locations separate alignment from Markdown indentation" { @@ -806,8 +1076,8 @@ test "syntax formatted locations separate alignment from Markdown indentation" { }; const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[std.mem.indexOf(u8, content, "Heading").?]); - try std.testing.expectEqual(@intFromEnum(Syn.string), styles[std.mem.indexOf(u8, content, "Code").?]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[std.mem.indexOf(u8, content, "Heading").?]); + try std.testing.expectEqual(@backingInt(Syn.string), styles[std.mem.indexOf(u8, content, "Code").?]); } fn inject(styles: []u8, source: []const u8, node: ts.Node, markdown: bool) void { const kind = node.kind(); @@ -858,35 +1128,273 @@ fn childOfKind(node: ts.Node, kind: []const u8) ?ts.Node { return null; } -pub fn highlightDiff(gpa: std.mem.Allocator, content: []const u8, start_byte_raw: usize, end_byte_raw: usize) ![]u8 { - const start_byte = @min(start_byte_raw, content.len); - const end_byte = @max(start_byte, @min(end_byte_raw, content.len)); - const source = content[start_byte..end_byte]; - const styles = try gpa.alloc(u8, source.len); +/// A diff line's style byte carries whether the line was added or removed +/// over its syntax style (the low bits), so the painter can tint its row +/// and mark its prefix; Diff marks the words a changed line changed +/// (hunks.zig), drawn on a stronger tint. +pub const diff_added: u8 = 0x80; +pub const diff_removed: u8 = 0x40; +pub const diff_emph: u8 = 0x20; +/// A +Diff line added or removed whole, paired with no line it changed: +/// its row takes a faint tint, on its marker's byte. +pub const diff_whole: u8 = 0x10; +pub const syn_bits: u8 = 0x0f; + +/// A file section's hunks are parsed together, each side as one text, in +/// pieces: a piece takes whole hunks until it holds `diff_piece_lines` +/// lines, and a hunk longer than `diff_piece_cut` is cut there, counted +/// from the section's first hunk. So a view parses about what a file's +/// view does (its rows and a margin), however many small hunks it shows, +/// and a 20k-line new file is parsed where it is shown, not whole. A string +/// or comment across a cut colours as a fragment would. +const diff_piece_lines = 40; +const diff_piece_cut = 80; + +/// Painted hunk pieces, keyed by their bytes and language: scrolling back +/// over a hunk, or a terminal repainting the same output every frame, +/// parses nothing again. Freed by `stop`. +const DiffPiece = struct { hash: u64, styles: []u8, used: u64 }; +var diff_pieces: [96]?DiffPiece = @splat(null); +var diff_clock: u64 = 0; + +fn dropDiffPieces() void { + for (&diff_pieces) |*slot| { + if (slot.*) |piece| syntax_allocator.free(piece.styles); + slot.* = null; + } + diff_clock = 0; +} + +/// The styles of `content[start..end)` read as a unified diff: headers +/// muted, `@@` lines as keywords, and each hunk's code in the language of +/// the file its section names. A hunk's old side (context and removed +/// lines) and new side (context and added lines) are each parsed whole, so +/// a string or comment that spans lines colours as it does in the file. +/// Added and removed lines carry `diff_added`/`diff_removed`. A file in no +/// language pardes knows keeps the plain colours: added lines as strings, +/// removed ones as numbers. +pub fn highlightDiff(gpa: std.mem.Allocator, content: []const u8, start_raw: usize, end_raw: usize) ![]u8 { + return highlightDiffRows(gpa, content, start_raw, end_raw, false); +} + +/// The same for a terminal's rows, where a row the terminal wrapped +/// continues the diff line above it. +pub fn highlightDiffRows(gpa: std.mem.Allocator, content: []const u8, start_raw: usize, end_raw: usize, continued_rows: bool) ![]u8 { + const tz = tracy.zone(@src(), "highlightDiff"); + defer tz.end(); + const start_byte = @min(start_raw, content.len); + const end_byte = @max(start_byte, @min(end_raw, content.len)); + const styles = try gpa.alloc(u8, end_byte - start_byte); errdefer gpa.free(styles); @memset(styles, 0); + const Window = struct { + styles: []u8, + start: usize, + fn put(w: @This(), at: usize, bytes: []const u8) void { + const lo = @max(at, w.start); + const hi = @min(at + bytes.len, w.start + w.styles.len); + if (lo < hi) @memcpy(w.styles[lo - w.start .. hi - w.start], bytes[lo - at .. hi - at]); + } + fn fill(w: @This(), at: usize, len: usize, style: u8) void { + const lo = @max(at, w.start); + const hi = @min(at + len, w.start + w.styles.len); + if (lo < hi) @memset(w.styles[lo - w.start .. hi - w.start], style); + } + }; + const window: Window = .{ .styles = styles, .start = start_byte }; + + var walk: diff.Walk = .{ .continued_rows = continued_rows }; + // The run of hunk lines not painted yet, with each line's kind. + var piece_start: ?usize = null; + var piece_end: usize = 0; + var piece_path: []const u8 = ""; + var kinds: [diff_piece_cut]diff.Kind = undefined; + var nkinds: usize = 0; + var at = diff.anchorBefore(content, start_byte); + while (true) { + const nl = std.mem.indexOfScalarPos(u8, content, at, '\n') orelse content.len; + const got = walk.step(content[at..nl]); + const body = switch (got.kind) { + .context, .added, .removed, .no_newline => true, + // The next hunk of the section goes on in the same piece. + .hunk => piece_start != null and nkinds < diff_piece_lines, + else => false, + }; + if (piece_start) |ps| if (!body or nkinds == diff_piece_cut) { + if (piece_end >= start_byte) try paintDiffPiece(gpa, window, content[ps..piece_end], ps, kinds[0..nkinds], piece_path); + piece_start = null; + nkinds = 0; + }; + if (at >= end_byte and piece_start == null) break; + if (body and (piece_start != null or got.kind != .hunk)) { + if (piece_start == null) { + piece_start = at; + piece_path = walk.path(); + } + kinds[nkinds] = got.kind; + nkinds += 1; + piece_end = nl; + } else if (nl >= start_byte) { + const style: Syn = switch (got.kind) { + .hunk => .keyword, + .meta, .old_path, .new_path => .comment, + else => .none, + }; + if (style != .none) window.fill(at, nl - at, @backingInt(style)); + } + if (nl >= content.len) break; + at = nl + 1; + } + if (piece_start) |ps| if (piece_end >= start_byte) + try paintDiffPiece(gpa, window, content[ps..piece_end], ps, kinds[0..nkinds], piece_path); + return styles; +} +fn paintDiffPiece(gpa: std.mem.Allocator, window: anytype, bytes: []const u8, at: usize, kinds: []const diff.Kind, path: []const u8) !void { + const selected: ?Selected = if (enabled and syntax_started) (forExt(std.fs.path.extension(path)) catch null) else null; + if (selected) |sel| { + const hash = std.hash.Wyhash.hash(std.hash.Wyhash.hash(0, sel.name), bytes); + diff_clock += 1; + for (&diff_pieces) |*slot| if (slot.*) |*piece| if (piece.hash == hash and piece.styles.len == bytes.len) { + piece.used = diff_clock; + return window.put(at, piece.styles); + }; + const colored = try syntax_allocator.alloc(u8, bytes.len); + errdefer syntax_allocator.free(colored); + try paintHunk(gpa, colored, bytes, kinds, sel); + window.put(at, colored); + // An empty slot, else the least recently used. + var oldest: usize = 0; + for (&diff_pieces, 0..) |*slot, i| { + const piece = slot.* orelse { + oldest = i; + break; + }; + if (piece.used < diff_pieces[oldest].?.used) oldest = i; + } + if (diff_pieces[oldest]) |old| syntax_allocator.free(old.styles); + diff_pieces[oldest] = .{ .hash = hash, .styles = colored, .used = diff_clock }; + return; + } + // No language: each line in one colour, as diffs were always shown. + var lines = std.mem.splitScalar(u8, bytes, '\n'); var offset: usize = 0; - var lines = std.mem.splitScalar(u8, source, '\n'); - while (lines.next()) |line| { - const syn = diffLineSyn(line); - if (syn != .none) @memset(styles[offset .. offset + line.len], @intFromEnum(syn)); + for (kinds) |kind| { + const line = lines.next() orelse break; + const style: u8 = switch (kind) { + .added => @backingInt(Syn.string) | diff_added, + .removed => @backingInt(Syn.number) | diff_removed, + .no_newline => @backingInt(Syn.comment), + .hunk => @backingInt(Syn.keyword), + else => 0, + }; + if (style != 0) window.fill(at + offset, line.len, style); offset += line.len + 1; } - return styles; } -fn diffLineSyn(line: []const u8) Syn { - if (std.mem.startsWith(u8, line, "@@")) return .keyword; - if (std.mem.startsWith(u8, line, "+++") or - std.mem.startsWith(u8, line, "---") or - std.mem.startsWith(u8, line, "diff ") or - std.mem.startsWith(u8, line, "index ") or - std.mem.startsWith(u8, line, "\\ No newline")) return .comment; - if (line.len == 0) return .none; - if (line[0] == '+') return .string; - if (line[0] == '-') return .number; - return .none; +/// One piece's styles: its old side and its new side each parsed as one +/// text, and every line's code coloured from its side, behind its +/// one-character prefix. A `@@` line between its hunks is in neither. +fn paintHunk(gpa: std.mem.Allocator, out: []u8, bytes: []const u8, kinds: []const diff.Kind, sel: Selected) !void { + @memset(out, 0); + const old_src = try gpa.alloc(u8, bytes.len); + defer gpa.free(old_src); + const new_src = try gpa.alloc(u8, bytes.len); + defer gpa.free(new_src); + var old_len: usize = 0; + var new_len: usize = 0; + var lines = std.mem.splitScalar(u8, bytes, '\n'); + for (kinds) |kind| { + const line = lines.next() orelse break; + const code = line[codeStart(line, kind)..]; + // A row a terminal wrapped goes on the line above, as it was printed. + const joined = continuesRow(line, kind); + if (kind == .context or kind == .removed) { + if (joined and old_len > 0) old_len -= 1; + @memcpy(old_src[old_len..][0..code.len], code); + old_len += code.len; + old_src[old_len] = '\n'; + old_len += 1; + } + if (kind == .context or kind == .added) { + if (joined and new_len > 0) new_len -= 1; + @memcpy(new_src[new_len..][0..code.len], code); + new_len += code.len; + new_src[new_len] = '\n'; + new_len += 1; + } + } + // A side is parsed only when a line takes its colours from it: the old + // side for removed lines, the new side for added lines, and context + // from the new side, else from the old. A piece that only adds or only + // removes is one parse. + var removes = false; + var adds = false; + for (kinds) |kind| { + removes = removes or kind == .removed; + adds = adds or kind == .added; + } + const context_new = adds or !removes; + const old_colors = try gpa.alloc(u8, old_len); + defer gpa.free(old_colors); + @memset(old_colors, 0); + if (!context_new or removes) paint(old_colors, old_src[0..old_len], sel); + const new_colors = try gpa.alloc(u8, new_len); + defer gpa.free(new_colors); + @memset(new_colors, 0); + if (context_new) paint(new_colors, new_src[0..new_len], sel); + + lines = std.mem.splitScalar(u8, bytes, '\n'); + var offset: usize = 0; + var old_at: usize = 0; + var new_at: usize = 0; + for (kinds) |kind| { + const line = lines.next() orelse break; + defer offset += line.len + 1; + const skip = codeStart(line, kind); + const code_len = line.len - skip; + const dst = out[offset + skip ..][0..code_len]; + if (continuesRow(line, kind)) { + if ((kind == .context or kind == .removed) and old_at > 0) old_at -= 1; + if ((kind == .context or kind == .added) and new_at > 0) new_at -= 1; + } + switch (kind) { + .removed => @memcpy(dst, old_colors[old_at..][0..code_len]), + .added => @memcpy(dst, new_colors[new_at..][0..code_len]), + .context => @memcpy(dst, if (context_new) new_colors[new_at..][0..code_len] else old_colors[old_at..][0..code_len]), + .no_newline => @memset(out[offset..][0..line.len], @backingInt(Syn.comment)), + .hunk => @memset(out[offset..][0..line.len], @backingInt(Syn.keyword)), + else => {}, + } + if (kind == .context or kind == .removed) old_at += code_len + 1; + if (kind == .context or kind == .added) new_at += code_len + 1; + const flag: u8 = switch (kind) { + .added => diff_added, + .removed => diff_removed, + else => 0, + }; + if (flag != 0) for (out[offset..][0..line.len]) |*b| { + b.* |= flag; + }; + } +} + +fn continuesRow(line: []const u8, kind: diff.Kind) bool { + return line.len > 0 and codeStart(line, kind) == 0 and + (kind == .context or kind == .added or kind == .removed); +} + +/// Where a hunk line's code starts: past its prefix, which a blank context +/// line (its space stripped) and a terminal's wrapped row have none of. +fn codeStart(line: []const u8, kind: diff.Kind) usize { + if (line.len == 0) return 0; + return switch (kind) { + .context => @intFromBool(line[0] == ' '), + .added => @intFromBool(line[0] == '+'), + .removed => @intFromBool(line[0] == '-'), + else => 0, + }; } test "syntax a results row is coloured by the file its location names" { @@ -904,8 +1412,8 @@ test "syntax a results row is coloured by the file its location names" { try std.testing.expectEqual(content.len, styles.len); for (styles[0.."src/a.zig:1:1".len]) |b| try std.testing.expectEqual(@as(u8, 0), b); const zig_kw = std.mem.indexOf(u8, content, "struct").?; - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[zig_kw]); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[zig_kw + 5]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[zig_kw]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[zig_kw + 5]); const prose = std.mem.indexOf(u8, content, "just some prose").?; for (styles[prose .. prose + 14]) |b| try std.testing.expectEqual(@as(u8, 0), b); const bare = std.mem.indexOf(u8, content, "src/c.zig").?; @@ -933,25 +1441,11 @@ test "syntax codeAfterLocation takes whole-token locations and nothing else" { try std.testing.expect(codeAfterLocation("src/x.zig:7:2") == null); try std.testing.expect(codeAfterLocation("") == null); try std.testing.expect(codeAfterLocation(" leading space") == null); -} - -test "syntax tree-sitter allocator callbacks preserve and free exact allocations" { - syntax_allocator = std.testing.allocator; - defer syntax_allocator = undefined; - - var live: ?*anyopaque = syntaxCalloc(4, 1) orelse return error.OutOfMemory; - defer if (live) |pointer| syntaxFree(pointer); - const original: [*]u8 = @ptrCast(live.?); - try std.testing.expectEqualSlices(u8, &.{ 0, 0, 0, 0 }, original[0..4]); - @memcpy(original[0..4], "data"); - - live = syntaxRealloc(live, 32) orelse return error.OutOfMemory; - const grown: [*]u8 = @ptrCast(live.?); - try std.testing.expectEqualSlices(u8, "data", grown[0..4]); - - try std.testing.expect(syntaxCalloc(std.math.maxInt(usize), 2) == null); - try std.testing.expect(syntaxRealloc(live, 0) == null); - live = null; + const indented = " some directory/x.zig:7:2-9 return 42;"; + const nested = codeAfterLocation(indented) orelse return error.ShouldBeALocation; + try std.testing.expectEqualStrings("some directory/x.zig", nested.path); + try std.testing.expectEqualStrings(" return 42;", nested.text); + try std.testing.expectEqualStrings(nested.text, indented[nested.at..]); } test "syntax default full grammar set highlights Typst source" { @@ -967,14 +1461,14 @@ test "syntax default full grammar set highlights Typst source" { const keyword_at = std.mem.indexOf(u8, source, "let").?; const number_at = std.mem.indexOf(u8, source, "42").?; const string_at = std.mem.indexOf(u8, source, "\"hello\"").?; - try std.testing.expectEqual(Syn.comment, @as(Syn, @enumFromInt(styles[comment_at]))); - try std.testing.expectEqual(Syn.keyword, @as(Syn, @enumFromInt(styles[keyword_at]))); - try std.testing.expectEqual(Syn.number, @as(Syn, @enumFromInt(styles[number_at]))); - try std.testing.expectEqual(Syn.string, @as(Syn, @enumFromInt(styles[string_at]))); + try std.testing.expectEqual(Syn.comment, @as(Syn, @fromBackingInt(@intCast(styles[comment_at])))); + try std.testing.expectEqual(Syn.keyword, @as(Syn, @fromBackingInt(@intCast(styles[keyword_at])))); + try std.testing.expectEqual(Syn.number, @as(Syn, @fromBackingInt(@intCast(styles[number_at])))); + try std.testing.expectEqual(Syn.string, @as(Syn, @fromBackingInt(@intCast(styles[string_at])))); const short_ext = try highlightFileRange(std.testing.allocator, "paper.typ", source, 0, source.len); defer std.testing.allocator.free(short_ext); - try std.testing.expectEqual(Syn.keyword, @as(Syn, @enumFromInt(short_ext[keyword_at]))); + try std.testing.expectEqual(Syn.keyword, @as(Syn, @fromBackingInt(@intCast(short_ext[keyword_at])))); } test "syntax Typst markup constructs paint and raw blocks inject their language" { @@ -999,7 +1493,7 @@ test "syntax Typst markup constructs paint and raw blocks inject their language" const synAt = struct { fn f(s: []const u8, src: []const u8, needle: []const u8, offset: usize) Syn { - return @enumFromInt(s[std.mem.indexOf(u8, src, needle).? + offset]); + return @fromBackingInt(@intCast(s[std.mem.indexOf(u8, src, needle).? + offset])); } }.f; try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "= Heading", 0)); @@ -1039,7 +1533,7 @@ test "syntax markdown highlights markup, injects inline spans and fenced code bl const synAt = struct { fn f(s: []const u8, src: []const u8, needle: []const u8, offset: usize) Syn { - return @enumFromInt(s[std.mem.indexOf(u8, src, needle).? + offset]); + return @fromBackingInt(@intCast(s[std.mem.indexOf(u8, src, needle).? + offset])); } }.f; try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "Title", 0)); @@ -1067,8 +1561,8 @@ test "syntax inline fast path agrees with full Markdown query" { const actual = try std.testing.allocator.alloc(u8, source.len); defer std.testing.allocator.free(actual); for ([_]Syn{ .none, .keyword }) |background| { - @memset(expected, @intFromEnum(background)); - @memset(actual, @intFromEnum(background)); + @memset(expected, @backingInt(background)); + @memset(actual, @backingInt(background)); runQuery(expected, sel, tree, 0); inject(actual, source, tree.rootNode(), true); if (!std.mem.eql(u8, expected, actual)) std.debug.print("inline mismatch: {s}\n", .{source}); @@ -1107,40 +1601,224 @@ test "syntax plain Markdown keeps block styles without starting the inline parse const source = "# Heading\n\nPlain prose.\n\n indented code\n"; const styles = try highlightFileRange(std.testing.allocator, "a.md", source, 0, source.len); defer std.testing.allocator.free(styles); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[2]); - try std.testing.expectEqual(@intFromEnum(Syn.none), styles[std.mem.indexOf(u8, source, "Plain").?]); - try std.testing.expectEqual(@intFromEnum(Syn.string), styles[std.mem.indexOf(u8, source, "indented").?]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[2]); + try std.testing.expectEqual(@backingInt(Syn.none), styles[std.mem.indexOf(u8, source, "Plain").?]); + try std.testing.expectEqual(@backingInt(Syn.string), styles[std.mem.indexOf(u8, source, "indented").?]); for (specs) |spec| { if (std.mem.eql(u8, spec.name, "markdown_inline")) try std.testing.expect(spec.selected == null); } } test "syntax highlightDiff colors unified diff lines by prefix" { - const diff = + const diff_text = "diff --git a/x b/x\n" ++ "--- a/x\n" ++ "+++ b/x\n" ++ - "@@ -1,3 +1,3 @@\n" ++ + "@@ -1,2 +1,2 @@\n" ++ " context\n" ++ "-old line\n" ++ "+new line\n"; - const styles = try highlightDiff(std.testing.allocator, diff, 0, diff.len); + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "diff --git", 0)); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "--- a/x", 0)); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "+++ b/x", 0)); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "@@ -1,2", 0)); + try std.testing.expectEqual(Syn.none, diffSynAt(styles, diff_text, " context", 0)); + // A file in no known language: each line in one colour, and tinted. + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "-old line", 0)); + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "+new line", 0)); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "-old line").?] & diff_removed != 0); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "+new line").? + 4] & diff_added != 0); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, " context").?] & (diff_added | diff_removed) == 0); +} - const byteSyn = struct { - fn at(s: []const u8, src: []const u8, needle: []const u8) Syn { - const i = std.mem.indexOf(u8, src, needle).?; - return @enumFromInt(s[i]); - } - }.at; +fn diffSynAt(styles: []const u8, text: []const u8, needle: []const u8, offset: usize) Syn { + return @fromBackingInt(@intCast(styles[std.mem.indexOf(u8, text, needle).? + offset] & syn_bits)); +} - try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "diff --git")); - try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "--- a/x")); - try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "+++ b/x")); - try std.testing.expectEqual(Syn.keyword, byteSyn(styles, diff, "@@ -1,3")); - try std.testing.expectEqual(Syn.none, byteSyn(styles, diff, " context")); - try std.testing.expectEqual(Syn.number, byteSyn(styles, diff, "-old line")); - try std.testing.expectEqual(Syn.string, byteSyn(styles, diff, "+new line")); +test "syntax a Zig hunk is coloured as Zig, each side parsed whole" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + const diff_text = + "diff --git a/src/shape.zig b/src/shape.zig\n" ++ + "index 1111111..2222222 100644\n" ++ + "--- a/src/shape.zig\n" ++ + "+++ b/src/shape.zig\n" ++ + "@@ -1,6 +1,6 @@\n" ++ + " const Shape = struct {\n" ++ + "- side: u8 = 7,\n" ++ + "+ side: u16 = 42,\n" ++ + " const note =\n" ++ + "- \\\\old words defer\n" ++ + "+ \\\\new words defer\n" ++ + " ;\n" ++ + " };\n"; + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "struct", 0)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "7,", 0)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "42", 0)); + // A multiline string, told apart only by parsing the side it is on: + // `defer` inside it is no keyword. + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "old words defer", 10)); + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "new words defer", 10)); + // The prefix is no code: it carries the tint alone. + const added = std.mem.indexOf(u8, diff_text, "+ side").?; + try std.testing.expectEqual(diff_added, styles[added]); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "42").?] & diff_added != 0); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "7,").?] & diff_removed != 0); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "index 111", 0)); + + // A window in the middle is coloured as the whole is. + const from = std.mem.indexOf(u8, diff_text, " const note").?; + const part = try highlightDiff(std.testing.allocator, diff_text, from, diff_text.len); + defer std.testing.allocator.free(part); + try std.testing.expectEqualSlices(u8, styles[from..], part); +} + +test "syntax a Python hunk is coloured as Python" { + if (!enabled or !full_grammars) return; + start(std.testing.allocator); + defer stop(); + const diff_text = + "--- a/tool.py\n" ++ + "+++ b/tool.py\n" ++ + "@@ -1,3 +1,4 @@\n" ++ + " def run(n):\n" ++ + "- return n\n" ++ + "+ # doubled now\n" ++ + "+ return n * 2\n" ++ + " \n"; + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "def", 0)); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "return n\n", 0)); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "# doubled", 2)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "2\n", 0)); +} + +test "syntax a deleted file's lines are coloured in its language" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + const diff_text = + "diff --git a/gone.zig b/gone.zig\n" ++ + "deleted file mode 100644\n" ++ + "--- a/gone.zig\n" ++ + "+++ /dev/null\n" ++ + "@@ -1,2 +0,0 @@\n" ++ + "-const Gone = enum { a };\n" ++ + "-// the end\n"; + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "enum", 0)); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "// the end", 3)); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "enum").?] & diff_removed != 0); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "deleted file", 0)); +} + +test "syntax a hunk in no known language keeps the line colours" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + const diff_text = + "--- a/notes.unknownext\n" ++ + "+++ b/notes.unknownext\n" ++ + "@@ -1 +1 @@\n" ++ + "-const struct 1\n" ++ + "+const struct 2\n"; + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "struct 1", 0)); + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "struct 2", 0)); + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "+const struct 2", 0)); +} + +test "syntax git diff output in a command pane is told and coloured by its rows" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + // A command pane's rows: its command line, then git's output, with one + // long added line the terminal wrapped onto a second row. + const rows = [_][]const u8{ + "git diff", + "diff --git a/main.zig b/main.zig", + "--- a/main.zig", + "+++ b/main.zig", + "@@ -1,2 +1,2 @@", + " pub fn main() void {", + "+ const answer: u32 = 42; // the ", + "wrapped rest", + "- return;", + "", + }; + try std.testing.expect(diff.looksLikeDiff(&rows)); + try std.testing.expect(!diff.looksLikeDiff(rows[5..])); + const text = try std.mem.join(std.testing.allocator, "\n", &rows); + defer std.testing.allocator.free(text); + const styles = try highlightDiffRows(std.testing.allocator, text, 0, text.len, true); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.none, diffSynAt(styles, text, "git diff", 0)); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, text, "fn main", 0)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, text, "42", 0)); + // The wrapped row stays an added line, and inside the comment. + try std.testing.expect(styles[std.mem.indexOf(u8, text, "wrapped rest").?] & diff_added != 0); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, text, "wrapped rest", 0)); + try std.testing.expect(styles[std.mem.indexOf(u8, text, "return").?] & diff_removed != 0); +} + +test "syntax a section's small hunks share one parse" { + if (!enabled) return; + const gpa = std.testing.allocator; + start(gpa); + defer stop(); + const diff_text = + "--- a/x.zig\n+++ b/x.zig\n" ++ + "@@ -1 +1 @@\n-const a = 1;\n+const a = 2;\n" ++ + "@@ -10 +10 @@\n-const b = 1;\n+const b = 2;\n" ++ + "@@ -20 +20 @@\n-const c = 1;\n+const c = 2;\n" ++ + "--- a/y.zig\n+++ b/y.zig\n" ++ + "@@ -1 +1 @@\n-const d = 1;\n+const d = 2;\n"; + const styles = try highlightDiff(gpa, diff_text, 0, diff_text.len); + defer gpa.free(styles); + var cached: usize = 0; + for (diff_pieces) |slot| cached += @intFromBool(slot != null); + try std.testing.expectEqual(@as(usize, 2), cached); // x.zig's three hunks, and y.zig's + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "@@ -10", 0)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "2;\n@@ -20", 0)); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "@@ -10").?] & (diff_added | diff_removed) == 0); +} + +test "syntax a hunk painted once is kept, and a long one is parsed in pieces" { + if (!enabled) return; + const gpa = std.testing.allocator; + start(gpa); + defer stop(); + var text: std.Io.Writer.Allocating = .init(gpa); + defer text.deinit(); + const n = diff_piece_cut * 20 + 10; + try text.writer.print("--- /dev/null\n+++ b/big.zig\n@@ -0,0 +1,{d} @@\n", .{n}); + for (0..n) |i| try text.writer.print("+const v{d} = {d};\n", .{ i, i }); + const content = text.written(); + const at = std.mem.indexOf(u8, content, "+const v1000 ").?; + const window = try highlightDiff(gpa, content, at, at + 200); + defer gpa.free(window); + try std.testing.expectEqual(Syn.number, @as(Syn, @fromBackingInt(@intCast(window["+const v1000 = ".len] & syn_bits)))); + var cached: usize = 0; + for (diff_pieces) |slot| if (slot) |piece| { + cached += 1; + // One piece of many, not the whole hunk. + try std.testing.expect(piece.styles.len < content.len / 4); + }; + try std.testing.expectEqual(@as(usize, 1), cached); + const again = try highlightDiff(gpa, content, at, at + 200); + defer gpa.free(again); + try std.testing.expectEqualSlices(u8, window, again); + cached = 0; + for (diff_pieces) |slot| cached += @intFromBool(slot != null); + try std.testing.expectEqual(@as(usize, 1), cached); } test "syntax result fragments preserve source indentation and inline markup" { @@ -1149,6 +1827,7 @@ test "syntax result fragments preserve source indentation and inline markup" { defer stop(); const fixtures = [_]struct { path: []const u8, source: []const u8 }{ .{ .path = "a.zig", .source = " const number = 42; // note" }, + .{ .path = " nested path/a.zig", .source = " return call(42, \"hello\"); // note" }, .{ .path = "a.md", .source = "# Heading *slant*" }, .{ .path = "a.md", .source = "**bold** and `code`" }, .{ .path = "a.md", .source = " # this is indented code" }, @@ -1191,7 +1870,7 @@ test "syntax query filtering preserves upstream colors" { reference.query = raw_query; reference.capture_styles = try std.testing.allocator.alloc(u8, raw_query.captureCount()); defer std.testing.allocator.free(reference.capture_styles); - for (reference.capture_styles, 0..) |*style, id| style.* = @intFromEnum(synFor(raw_query.captureNameForId(@intCast(id)) orelse "")); + for (reference.capture_styles, 0..) |*style, id| style.* = @backingInt(synFor(raw_query.captureNameForId(@intCast(id)) orelse "")); const tree = selected.parser.parseString(source, null) orelse return error.ParseFailed; defer tree.destroy(); var expected: [source.len]u8 = @splat(0); @@ -1217,8 +1896,8 @@ test "syntax Zig keyword captures cover both bytes beyond line ten thousand" { defer std.testing.allocator.free(styles); for ([_][]const u8{ "fn", "if" }) |keyword| { const at = std.mem.indexOf(u8, code, keyword).?; - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[at]); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[at + 1]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[at]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[at + 1]); } } @@ -1234,8 +1913,8 @@ test "syntax allocator switching clears default-runtime caches" { const restored = try highlightFileRange(std.testing.allocator, "a.zig", source, 0, source.len); defer std.testing.allocator.free(restored); defer stop(); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), restored[0]); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), restored[1]); + try std.testing.expectEqual(@backingInt(Syn.keyword), restored[0]); + try std.testing.expectEqual(@backingInt(Syn.keyword), restored[1]); } test "syntax location context snapshots retain omitted multiline comment scope" { @@ -1259,8 +1938,8 @@ test "syntax location context snapshots retain omitted multiline comment scope" const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); for (styles[rows[0].code_start..][0..code.len]) |style| - try std.testing.expectEqual(@intFromEnum(Syn.comment), style); - try std.testing.expectEqual(@intFromEnum(Syn.none), styles[0]); + try std.testing.expectEqual(@backingInt(Syn.comment), style); + try std.testing.expectEqual(@backingInt(Syn.none), styles[0]); } test "syntax declaration locations clear snapshots without muting ordinary context" { @@ -1270,7 +1949,7 @@ test "syntax declaration locations clear snapshots without muting ordinary conte defer stop(); const header = "pub fn run() void {"; const content = "| \t" ++ header ++ "\n| \t const value = 42;\nfile.zig:3\t}\n"; - const snapshots = [_]u8{0xff} ** header.len; + const snapshots = @as([header.len]u8, @splat(0xff)); const Row = struct { path: []const u8 = "file.zig", code_start: usize, @@ -1286,7 +1965,7 @@ test "syntax declaration locations clear snapshots without muting ordinary conte const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); for (styles[3..][0..header.len]) |style| try std.testing.expectEqual(@as(u8, 0), style); - try std.testing.expectEqual(@intFromEnum(Syn.number), styles[std.mem.indexOf(u8, content, "42").?]); + try std.testing.expectEqual(@backingInt(Syn.number), styles[std.mem.indexOf(u8, content, "42").?]); } test "syntax stacked addresses do not split multiline source groups" { @@ -1306,11 +1985,11 @@ test "syntax stacked addresses do not split multiline source groups" { }; const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); - try std.testing.expectEqual(@intFromEnum(Syn.comment), styles[std.mem.indexOf(u8, content, "42").?]); - try std.testing.expectEqual(@intFromEnum(Syn.number), styles[std.mem.indexOf(u8, content, "7;").?]); + try std.testing.expectEqual(@backingInt(Syn.comment), styles[std.mem.indexOf(u8, content, "42").?]); + try std.testing.expectEqual(@backingInt(Syn.number), styles[std.mem.indexOf(u8, content, "7;").?]); for ([_][]const u8{ "a.c:1:1", "a.c:2:1", "a.c:3:1" }) |label| { const at = std.mem.indexOf(u8, content, label).?; - for (styles[at..][0..label.len]) |style| try std.testing.expectEqual(@intFromEnum(Syn.none), style); + for (styles[at..][0..label.len]) |style| try std.testing.expectEqual(@backingInt(Syn.none), style); } } @@ -1374,3 +2053,72 @@ test "syntax source analysis preserves declarations and injected colors" { try std.testing.expectEqual(@as(usize, 0), unsupported.declarations.len); try std.testing.expectEqual(@as(usize, 0), unsupported.colors.len); } + +test "tables are the pipe_tables tree-sitter finds" { + if (!enabled or !full_grammars) return; + start(std.testing.allocator); + defer stop(); + const Tables = @import("Tables.zig"); + const source = + "# Tables\n" ++ + "\n" ++ + "| a | b |\n" ++ + "|---|:-:|\n" ++ + "| 1 | 2 |\n" ++ + "no pipe row\n" ++ + "\n" ++ + "a | b\n" ++ + "--|--\n" ++ + "c | d\n" ++ + "# heading ends it\n" ++ + "| x |\n" ++ + "|---|\n" ++ + "\n" ++ + "```\n" ++ + "| in | fence |\n" ++ + "|----|-------|\n" ++ + "```\n" ++ + "\n" ++ + "| one | two |\n" ++ + "|-----|\n" ++ + "\n" ++ + " | indented | table |\n" ++ + " | -------- | ----- |\n" ++ + " | row | here |\n" ++ + "> quote ends\n" ++ + "\n" ++ + "| esc \\| aped | two |\n" ++ + "| --- | --- |\n" ++ + "- list ends\n"; + const selected = (try forLang("markdown")).?; + const tree = selected.parser.parseString(source, null) orelse return error.ParseFailed; + defer tree.destroy(); + // Every pipe_table node's lines, wherever it is in the tree. + var found: std.ArrayList([2]u32) = .empty; + defer found.deinit(std.testing.allocator); + var stack: std.ArrayList(ts.Node) = .empty; + defer stack.deinit(std.testing.allocator); + try stack.append(std.testing.allocator, tree.rootNode()); + while (stack.pop()) |node| { + if (std.mem.eql(u8, node.kind(), "pipe_table")) { + const end = node.endPoint(); + try found.append(std.testing.allocator, .{ node.startPoint().row, end.row + @intFromBool(end.column > 0) }); + continue; + } + var i = node.childCount(); + while (i > 0) : (i -= 1) if (node.child(i - 1)) |child| try stack.append(std.testing.allocator, child); + } + // The tables, the indented one and the escaped pipe's; not the fence's + // or the one whose delimiter has a cell too few. + try std.testing.expectEqual(@as(usize, 5), found.items.len); + var t: Tables = .{}; + defer t.deinit(std.testing.allocator); + try t.scan(std.testing.allocator, source); + var mine: std.ArrayList([2]u32) = .empty; + defer mine.deinit(std.testing.allocator); + for (t.list.items) |table| try mine.append(std.testing.allocator, .{ table.first, table.end }); + std.testing.expectEqualSlices([2]u32, found.items, mine.items) catch |e| { + std.debug.print("tree-sitter {any}\nTables {any}\n", .{ found.items, mine.items }); + return e; + }; +} |
