diff options
Diffstat (limited to 'src/syntax.zig')
| -rw-r--r-- | src/syntax.zig | 1090 |
1 files changed, 919 insertions, 171 deletions
diff --git a/src/syntax.zig b/src/syntax.zig index e637bf67..985a9888 100644 --- a/src/syntax.zig +++ b/src/syntax.zig @@ -3,6 +3,13 @@ const config = @import("pardes_config"); const tracy = @import("tracy.zig"); const grammar_manifest = @import("grammar_manifest.zig"); const look = @import("look.zig"); +const regexp = @import("regexp.zig"); +const c_heap = @import("c_heap"); +const diff = @import("diff.zig"); + +test { + _ = diff; +} pub const enabled = config.syntax_highlighting; const zig_grammar = config.syntax_zig_grammar; @@ -26,6 +33,10 @@ const Spec = struct { exts: []const []const u8, language: *const fn () callconv(.c) *const ts.Language, query_src: []const u8, + /// helix's textobjects.scm for the grammar (vendor/queries), "" for none + objects_src: []const u8 = "", + objects: ?*ts.Query = null, + objects_failed: bool = false, selected: ?Selected = null, capture_styles: [256]u8 = undefined, }; @@ -130,6 +141,7 @@ fn initSpecs() [specCount()]Spec { .exts = g.exts, .language = @extern(LanguageFn, .{ .name = "tree_sitter_" ++ g.name }), .query_src = querySrc(g), + .objects_src = @field(ts_queries, g.name ++ "_textobjects"), }; i += 1; } @@ -139,8 +151,11 @@ fn initSpecs() [specCount()]Spec { var specs = initSpecs(); -const allocation_header_size = 16; +/// What tree-sitter allocates from, the runtime and every grammar's external +/// scanner (build.zig compiles them to use the runtime's allocator): +/// `start`'s allocator, through the shared C heap. var syntax_allocator: std.mem.Allocator = undefined; +const syntax_heap = c_heap.Heap(&syntax_allocator); var syntax_started = false; extern fn ts_set_allocator( @@ -150,56 +165,13 @@ extern fn ts_set_allocator( new_free: ?*const fn (ptr: ?*anyopaque) callconv(.c) void, ) void; -fn syntaxAlloc(size_arg: usize) callconv(.c) ?*anyopaque { - const size = @max(size_arg, 1); - const total = std.math.add(usize, allocation_header_size, size) catch return null; - const bytes = syntax_allocator.alignedAlloc(u8, .@"16", total) catch return null; - const header: *align(16) usize = @ptrCast(bytes.ptr); - header.* = total; - return @ptrCast(bytes.ptr + allocation_header_size); -} - -fn syntaxCalloc(count: usize, size: usize) callconv(.c) ?*anyopaque { - const len = std.math.mul(usize, count, size) catch return null; - const pointer = syntaxAlloc(len) orelse return null; - const bytes: [*]u8 = @ptrCast(pointer); - @memset(bytes[0..len], 0); - return pointer; -} - -fn syntaxRealloc(ptr: ?*anyopaque, new_size: usize) callconv(.c) ?*anyopaque { - const pointer = ptr orelse return syntaxAlloc(new_size); - if (new_size == 0) { - syntaxFree(pointer); - return null; - } - const user: [*]u8 = @ptrCast(pointer); - const header: *align(16) usize = @ptrCast(@alignCast(user - allocation_header_size)); - const old_total = header.*; - const old_bytes: []align(16) u8 = @as([*]align(16) u8, @ptrCast(header))[0..old_total]; - const new_total = std.math.add(usize, allocation_header_size, new_size) catch return null; - const new_bytes = syntax_allocator.realloc(old_bytes, new_total) catch return null; - const new_header: *align(16) usize = @ptrCast(new_bytes.ptr); - new_header.* = new_total; - return @ptrCast(new_bytes.ptr + allocation_header_size); -} - -fn syntaxFree(ptr: ?*anyopaque) callconv(.c) void { - const pointer = ptr orelse return; - const user: [*]u8 = @ptrCast(pointer); - const header: *align(16) usize = @ptrCast(@alignCast(user - allocation_header_size)); - const total = header.*; - const bytes: []align(16) u8 = @as([*]align(16) u8, @ptrCast(header))[0..total]; - syntax_allocator.free(bytes); -} - pub fn start(gpa: std.mem.Allocator) void { if (comptime enabled) { std.debug.assert(!syntax_started); stop(); syntax_allocator = gpa; syntax_started = true; - ts_set_allocator(syntaxAlloc, syntaxCalloc, syntaxRealloc, syntaxFree); + ts_set_allocator(syntax_heap.malloc, syntax_heap.calloc, syntax_heap.realloc, syntax_heap.free); } } @@ -212,7 +184,11 @@ pub fn stop() void { selected.query.destroy(); } spec.selected = null; + if (spec.objects) |q| q.destroy(); + spec.objects = null; + spec.objects_failed = false; } + dropDiffPieces(); ts_set_allocator(null, null, null, null); syntax_started = false; syntax_allocator = undefined; @@ -238,7 +214,7 @@ fn ensure(spec: *Spec) !Selected { const capture_styles = spec.capture_styles[0..query.captureCount()]; for (capture_styles, 0..) |*style, id| { const name = query.captureNameForId(@intCast(id)) orelse ""; - style.* = @intFromEnum(synFor(name)); + style.* = @backingInt(synFor(name)); if (style.* == 0) query.disableCapture(name); } const parser = ts.Parser.create(); @@ -265,35 +241,8 @@ fn forExt(ext: []const u8) !?Selected { return null; } -const LangAlias = struct { []const u8, []const u8 }; -const lang_aliases = [_]LangAlias{ - .{ "js", "javascript" }, - .{ "jsx", "javascript" }, - .{ "mjs", "javascript" }, - .{ "py", "python" }, - .{ "sh", "bash" }, - .{ "shell", "bash" }, - .{ "zsh", "bash" }, - .{ "bash", "bash" }, - .{ "rs", "rust" }, - .{ "c++", "cpp" }, - .{ "cxx", "cpp" }, - .{ "cc", "cpp" }, - .{ "cs", "c_sharp" }, - .{ "kt", "kotlin" }, - .{ "rb", "ruby" }, - .{ "ml", "ocaml" }, - .{ "hs", "haskell" }, -}; - fn forLang(name: []const u8) !?Selected { - var canonical = name; - for (lang_aliases) |a| { - if (std.ascii.eqlIgnoreCase(name, a[0])) { - canonical = a[1]; - break; - } - } + const canonical = if (grammar_manifest.byName(name)) |i| grammar_manifest.all[i].name else name; for (&specs) |*spec| { if (std.ascii.eqlIgnoreCase(canonical, spec.name)) return try ensure(spec); } @@ -355,6 +304,56 @@ pub fn highlightFileRange(gpa: std.mem.Allocator, path: []const u8, content: []c return styles; } +/// Whether `highlightApart` can colour `path` on a worker: a grammar with +/// no injections (markdown's and typst's reach the shared parsers), made +/// ready here, on the loop, so the worker only reads its query. +pub fn readyApart(path: []const u8) bool { + if (!enabled) return false; + const selected = (forExt(std.fs.path.extension(path)) catch return false) orelse return false; + return !std.mem.eql(u8, selected.name, "markdown") and !std.mem.eql(u8, selected.name, "typst"); +} + +/// `highlightFileRange` of `source` (a copy of the range), on a worker: a +/// parser and a query cursor of its own, the grammar's query shared (it +/// is only read); `halt` ends the parse early. After `readyApart`. +pub fn highlightApart(gpa: std.mem.Allocator, path: []const u8, source: []const u8, halt: *const std.atomic.Value(bool)) ![]u8 { + if (comptime !enabled) return &.{}; + const shared = (try forExt(std.fs.path.extension(path))) orelse return &.{}; + const parser = ts.Parser.create(); + defer parser.destroy(); + try parser.setLanguage(shared.lang); + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + const Reader = struct { + text: []const u8, + fn read(payload: ?*anyopaque, byte: u32, _: ts.Point, n: *u32) callconv(.c) [*c]const u8 { + const r: *@This() = @ptrCast(@alignCast(payload.?)); + const at = @min(byte, r.text.len); + n.* = @intCast(r.text.len - at); + return r.text.ptr + at; + } + fn progress(state: *ts.Parser.State) callconv(.c) bool { + const flag: *const std.atomic.Value(bool) = @ptrCast(@alignCast(state.payload.?)); + return flag.load(.monotonic); + } + }; + var reader: Reader = .{ .text = source }; + const tree = parser.parseWithOptions( + .{ .payload = &reader, .read = Reader.read, .encoding = .utf8 }, + null, + .{ .payload = @constCast(halt), .progress_callback = @ptrCast(&Reader.progress) }, + ) orelse return error.Canceled; + defer tree.destroy(); + if (halt.load(.monotonic)) return error.Canceled; + const styles = try gpa.alloc(u8, source.len); + @memset(styles, 0); + var own = shared; + own.parser = parser; + own.cursor = cursor; + runQuery(styles, own, tree, 0); + return styles; +} + /// Source spans are independent of editor rows. Locations output uses the same /// declarations as sticky editor headers. Byte ends are exclusive; rows are /// zero-based and inclusive, including the last signature/opening-brace row. @@ -376,6 +375,233 @@ pub fn supportsPath(path: []const u8) bool { return false; } +/// A whole file's parse, kept by the file for the keys that walk its nodes +/// (helix's Alt-o and kin). Opaque outside this file; null without a grammar. +pub fn parseTree(path: []const u8, content: []const u8) ?*anyopaque { + if (!enabled) return null; + const selected = (forExt(std.fs.path.extension(path)) catch return null) orelse return null; + const tree = selected.parser.parseString(content, null) orelse return null; + return @ptrCast(tree); +} + +pub fn freeTree(tree: *anyopaque) void { + if (!enabled) return; + const t: *ts.Tree = @ptrCast(@alignCast(tree)); + t.destroy(); +} + +/// Where a node walk goes from a range (helix object.rs and movement.rs). +pub const NodeWalk = @import("modal.zig").Normal.NodeWalk; +pub const Span = struct { from: usize, to: usize }; + +/// The spans `walk` takes the byte range [from, to) to, into `out`; `at` is +/// the range's cursor. None when the tree has nothing to say there. +pub fn walkNodes(tree_ptr: *anyopaque, walk: NodeWalk, from: usize, to: usize, at: usize, out: []Span) usize { + if (!enabled or out.len == 0) return 0; + const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr)); + const lo: u32 = @intCast(@min(from, std.math.maxInt(u32))); + const hi: u32 = @intCast(@min(to, std.math.maxInt(u32))); + if (walk == .parent_end or walk == .parent_start) { + var node = tree.rootNode().namedDescendantForByteRange(lo, hi) orelse return 0; + if (walk == .parent_end) { + out[0] = .{ .from = node.endByte(), .to = node.endByte() }; + return 1; + } + // already at the node's lo: its first ancestor that starts earlier + if (node.startByte() == at) { + const first = node.startByte(); + while (node.startByte() >= first or !node.isNamed()) node = node.parent() orelse break; + } + out[0] = .{ .from = node.startByte(), .to = node.startByte() }; + return 1; + } + var cursor = tree.walk(); + defer cursor.destroy(); + // helix TreeCursor.reset_to_byte_range: the smallest node holding the range + while (true) { + const node = cursor.node(); + if (lo < node.startByte() or hi > node.endByte()) { + _ = cursor.gotoParent(); + break; + } + if (cursor.gotoFirstChildForByte(lo) == null) break; + } + switch (walk) { + .expand => while (cursor.node().startByte() == lo and cursor.node().endByte() == hi) { + if (!cursor.gotoParent()) break; + }, + .shrink => _ = cursor.gotoFirstChild(), + .next_sibling => while (!cursor.gotoNextSibling()) { + if (!cursor.gotoParent()) break; + }, + .prev_sibling => while (!cursor.gotoPreviousSibling()) { + if (!cursor.gotoParent()) break; + }, + .all_siblings, .all_children => { + if (walk == .all_siblings) { + while (true) { + if (!cursor.gotoParent()) return 0; + if (cursor.node().childCount() > 1) break; + } + } + // the named children, or nothing to say + var n: usize = 0; + if (!cursor.gotoFirstChild()) return 0; + while (true) { + const child = cursor.node(); + if (child.isNamed() and n < out.len) { + out[n] = .{ .from = child.startByte(), .to = child.endByte() }; + n += 1; + } + if (!cursor.gotoNextSibling()) break; + } + return n; + }, + .parent_end, .parent_start => unreachable, + } + const node = cursor.node(); + out[0] = .{ .from = node.startByte(), .to = node.endByte() }; + return 1; +} + +/// helix's textobjects, which its `]f`, `mif` and kin select: the name a +/// query's captures begin with (`function.around`, `test.inside`). +pub const Object = @import("modal.zig").Normal.Object; + +fn objectName(o: Object) []const u8 { + return if (o == .xml_element) "xml-element" else @tagName(o); +} + +/// The grammar's textobject query for `path`, compiled the first time; null +/// when the grammar has none. +fn objectQuery(path: []const u8) ?*ts.Query { + const ext = std.fs.path.extension(path); + for (&specs) |*spec| for (spec.exts) |choice| { + if (!std.ascii.eqlIgnoreCase(ext, choice)) continue; + if (spec.objects) |q| return q; + if (spec.objects_src.len == 0 or spec.objects_failed) return null; + var error_offset: u32 = 0; + spec.objects = ts.Query.create(spec.language(), spec.objects_src, &error_offset) catch { + spec.objects_failed = true; + return null; + }; + return spec.objects; + }; + return null; +} + +fn captureId(q: *const ts.Query, name: []const u8) ?u32 { + for (0..q.captureCount()) |i| if (std.mem.eql(u8, q.captureNameForId(@intCast(i)) orelse "", name)) return @intCast(i); + return null; +} + +/// A match's `capture`, as one span over all its nodes (helix's grouped +/// capture), when the match's `#eq?` and `#match?` hold. +fn matchSpan(q: *const ts.Query, match: anytype, capture: u32, text: []const u8) ?Span { + var span: ?Span = null; + for (match.captures) |c| if (c.index == capture) { + const s: Span = .{ .from = c.node.startByte(), .to = c.node.endByte() }; + span = if (span) |old| .{ .from = @min(old.from, s.from), .to = @max(old.to, s.to) } else s; + }; + var found = span orelse return null; + // a node that takes its line's newline (this zig grammar's comments) + // is the line without it, as helix's grammars have it + if (found.to > found.from + 1 and found.to <= text.len and text[found.to - 1] == '\n') found.to -= 1; + const steps = q.predicatesForPattern(match.pattern_index); + var i: usize = 0; + while (i < steps.len) { + var end = i; + while (end < steps.len and steps[end].type != .done) end += 1; + const pred = steps[i..end]; + i = end + 1; + if (pred.len != 3 or pred[0].type != .string or pred[1].type != .capture) continue; + const op = q.stringValueForId(pred[0].value_id) orelse continue; + const subject = for (match.captures) |c| { + if (c.index == pred[1].value_id) break text[@min(c.node.startByte(), text.len)..@min(c.node.endByte(), text.len)]; + } else continue; + const want = if (pred[2].type == .string) q.stringValueForId(pred[2].value_id) orelse continue else for (match.captures) |c| { + if (c.index == pred[2].value_id) break text[@min(c.node.startByte(), text.len)..@min(c.node.endByte(), text.len)]; + } else continue; + if (std.mem.eql(u8, op, "eq?")) { + if (!std.mem.eql(u8, subject, want)) return null; + } else if (std.mem.eql(u8, op, "match?")) { + var re = regexp.Regex.compile(want) catch continue; + if ((re.find(subject, 0, subject.len, subject.len) catch null) == null) return null; + } + } + return found; +} + +/// `mi`/`ma`: the smallest `object.inside` (or `.around`) holding byte `at` +/// (helix textobject_treesitter). None without a query or such an object. +pub fn objectAt(path: []const u8, tree_ptr: *anyopaque, text: []const u8, object: Object, around: bool, at: usize) ?Span { + if (!enabled) return null; + const q = objectQuery(path) orelse return null; + var buf: [32]u8 = undefined; + const want = std.fmt.bufPrint(&buf, "{s}.{s}", .{ objectName(object), if (around) "around" else "inside" }) catch return null; + const capture = captureId(q, want) orelse return null; + const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr)); + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + cursor.exec(q, tree.rootNode()); + var best: ?Span = null; + while (cursor.nextMatch()) |match| { + const s = matchSpan(q, match, capture, text) orelse continue; + if (!(s.from <= at and at < s.to)) continue; + if (best == null or s.to - s.from < best.?.to - best.?.from) best = s; + } + const s = best orelse return null; + if (s.from >= text.len or s.to >= text.len) return null; + return s; +} + +/// `]f`/`[f` and kin: the next object starting after byte `at`, or the +/// previous one ending before it (helix goto_treesitter_object), from the +/// first of `.movement`, `.around`, `.inside` the query names. +pub fn objectNext(path: []const u8, tree_ptr: *anyopaque, text: []const u8, object: Object, forward: bool, at: usize) ?Span { + if (!enabled) return null; + const q = objectQuery(path) orelse return null; + var buf: [32]u8 = undefined; + const capture = for ([_][]const u8{ "movement", "around", "inside" }) |kind| { + const want = std.fmt.bufPrint(&buf, "{s}.{s}", .{ objectName(object), kind }) catch return null; + if (captureId(q, want)) |id| break id; + } else return null; + const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr)); + const cursor = ts.QueryCursor.create(); + defer cursor.destroy(); + cursor.exec(q, tree.rootNode()); + var best: ?Span = null; + while (cursor.nextMatch()) |match| { + const s = matchSpan(q, match, capture, text) orelse continue; + if (forward) { + if (s.from <= at) continue; + if (best == null or s.from < best.?.from or (s.from == best.?.from and s.to > best.?.to)) best = s; + } else { + if (s.to >= at) continue; + if (best == null or s.to > best.?.to or (s.to == best.?.to and s.from < best.?.from)) best = s; + } + } + const s = best orelse return null; + if (s.from >= text.len or s.to >= text.len) return null; + return s; +} + +test "every vendored textobject query compiles against its grammar" { + if (!enabled) return; + var failed: usize = 0; + for (&specs) |*spec| { + if (spec.objects_src.len == 0) continue; + var error_offset: u32 = 0; + const q = ts.Query.create(spec.language(), spec.objects_src, &error_offset) catch { + std.debug.print("textobjects for {s} do not compile (offset {d})\n", .{ spec.name, error_offset }); + failed += 1; + continue; + }; + q.destroy(); + } + try std.testing.expectEqual(@as(usize, 0), failed); +} + /// Owned source analysis. Both slices use the allocator passed to analyzeSource. /// The parsed tree is released before returning; callers can cache this result. pub const SourceAnalysis = struct { @@ -396,7 +622,7 @@ pub fn analyzeSource(gpa: std.mem.Allocator, path: []const u8, content: []const defer tree.destroy(); var result: SourceAnalysis = .{}; errdefer result.deinit(gpa); - if (want_declarations) result.declarations = try declarationsFromTree(gpa, content, tree); + if (want_declarations) result.declarations = try declarationsFromTree(gpa, content, tree, null); if (want_colors) { result.colors = try gpa.alloc(u8, content.len); @memset(result.colors, 0); @@ -410,14 +636,55 @@ pub fn contextDeclarations(gpa: std.mem.Allocator, path: []const u8, content: [] return analysis.declarations; } -fn declarationsFromTree(gpa: std.mem.Allocator, content: []const u8, tree: *ts.Tree) ![]ContextDeclaration { +/// The declarations of `content` (its path's language), parsed with a +/// parser of its own, so a worker thread may ask: the loop's parsers +/// (`ensure`'s, one a language) are not touched, and a language is safe to +/// share. `stop`, set from another thread, ends the parse and the walk +/// early (error.Canceled): a 1.2 MB file is seconds in a Debug build. +pub fn declarationsApart(gpa: std.mem.Allocator, path: []const u8, content: []const u8, halt: *const std.atomic.Value(bool)) ![]ContextDeclaration { + if (comptime !enabled) return &.{}; + if (content.len == 0) return &.{}; + const ext = std.fs.path.extension(path); + const spec = specs: for (&specs) |*spec| { + for (spec.exts) |choice| if (std.ascii.eqlIgnoreCase(ext, choice)) break :specs spec; + } else return &.{}; + const parser = ts.Parser.create(); + defer parser.destroy(); + try parser.setLanguage(spec.language()); + const Reader = struct { + text: []const u8, + fn read(payload: ?*anyopaque, byte: u32, _: ts.Point, n: *u32) callconv(.c) [*c]const u8 { + const r: *@This() = @ptrCast(@alignCast(payload.?)); + const at = @min(byte, r.text.len); + n.* = @intCast(r.text.len - at); + return r.text.ptr + at; + } + fn progress(state: *ts.Parser.State) callconv(.c) bool { + const flag: *const std.atomic.Value(bool) = @ptrCast(@alignCast(state.payload.?)); + return flag.load(.monotonic); + } + }; + var reader: Reader = .{ .text = content }; + const tree = parser.parseWithOptions( + .{ .payload = &reader, .read = Reader.read, .encoding = .utf8 }, + null, + .{ .payload = @constCast(halt), .progress_callback = @ptrCast(&Reader.progress) }, + ) orelse return error.Canceled; + defer tree.destroy(); + return declarationsFromTree(gpa, content, tree, halt); +} + +fn declarationsFromTree(gpa: std.mem.Allocator, content: []const u8, tree: *ts.Tree, halt: ?*const std.atomic.Value(bool)) ![]ContextDeclaration { var result: std.ArrayList(ContextDeclaration) = .empty; errdefer result.deinit(gpa); // The cursor retains its ancestor stack: sibling visits do not repeatedly // reconstruct parents in broad syntax trees. Preorder preserves source order. var cursor = tree.rootNode().walk(); defer cursor.destroy(); + var visited: usize = 0; while (true) { + visited += 1; + if (halt) |flag| if (visited % 4096 == 0 and flag.load(.monotonic)) return error.Canceled; const node = cursor.node(); if (node.isNamed()) if (contextSpan(node, content)) |span| { if (result.items.len == 0 or result.items[result.items.len - 1].start_line != span.start_line or @@ -677,7 +944,13 @@ pub fn highlightLocationRows(gpa: std.mem.Allocator, content: []const u8, rows: if (comptime @hasField(@TypeOf(row), "location_end")) { address_only = row.location_end > 0 and source_start == end; } - if (!address_only) { + // A row its producer coloured whole is coloured below: no + // fragment of it to parse. + var coloured = false; + if (comptime @hasField(@TypeOf(row), "colors")) { + coloured = row.colors.len > 0 and row.colors.len >= end - source_start; + } + if (!address_only and !coloured) { sources[source_count] = .{ .index = index, .start = source_start, .len = end - source_start }; source_count += 1; } @@ -757,13 +1030,10 @@ pub fn highlightLocationRows(gpa: std.mem.Allocator, content: []const u8, rows: } fn codeAfterLocation(line: []const u8) ?struct { path: []const u8, at: usize, text: []const u8 } { - const token_end = std.mem.indexOfAny(u8, line, " \t") orelse return null; - if (token_end == 0) return null; - const token = line[0..token_end]; - const target = look.parsePathLine(token); - if (target.end != token.len) return null; + const trimmed = std.mem.trimStart(u8, line, " \t"); + const target = @import("locations.zig").parseRaw(trimmed); if (target.at.line == 0) return null; - const at = token_end + 1; + const at = line.len - trimmed.len + target.code_start; if (at >= line.len) return null; return .{ .path = target.path, .at = at, .text = line[at..] }; } @@ -786,11 +1056,11 @@ test "syntax formatted locations color hidden context and preserve multiline sou const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); const body = std.mem.indexOf(u8, content, "comment body").?; - try std.testing.expectEqual(@intFromEnum(Syn.comment), styles[body]); - try std.testing.expectEqual(@intFromEnum(Syn.none), styles[body - 1]); + try std.testing.expectEqual(@backingInt(Syn.comment), styles[body]); + try std.testing.expectEqual(@backingInt(Syn.none), styles[body - 1]); const number = std.mem.indexOf(u8, content, "42").?; - try std.testing.expectEqual(@intFromEnum(Syn.number), styles[number]); - try std.testing.expectEqual(@intFromEnum(Syn.none), styles[0]); + try std.testing.expectEqual(@backingInt(Syn.number), styles[number]); + try std.testing.expectEqual(@backingInt(Syn.none), styles[0]); } test "syntax formatted locations separate alignment from Markdown indentation" { @@ -806,8 +1076,8 @@ test "syntax formatted locations separate alignment from Markdown indentation" { }; const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[std.mem.indexOf(u8, content, "Heading").?]); - try std.testing.expectEqual(@intFromEnum(Syn.string), styles[std.mem.indexOf(u8, content, "Code").?]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[std.mem.indexOf(u8, content, "Heading").?]); + try std.testing.expectEqual(@backingInt(Syn.string), styles[std.mem.indexOf(u8, content, "Code").?]); } fn inject(styles: []u8, source: []const u8, node: ts.Node, markdown: bool) void { const kind = node.kind(); @@ -858,35 +1128,273 @@ fn childOfKind(node: ts.Node, kind: []const u8) ?ts.Node { return null; } -pub fn highlightDiff(gpa: std.mem.Allocator, content: []const u8, start_byte_raw: usize, end_byte_raw: usize) ![]u8 { - const start_byte = @min(start_byte_raw, content.len); - const end_byte = @max(start_byte, @min(end_byte_raw, content.len)); - const source = content[start_byte..end_byte]; - const styles = try gpa.alloc(u8, source.len); +/// A diff line's style byte carries whether the line was added or removed +/// over its syntax style (the low bits), so the painter can tint its row +/// and mark its prefix; Diff marks the words a changed line changed +/// (hunks.zig), drawn on a stronger tint. +pub const diff_added: u8 = 0x80; +pub const diff_removed: u8 = 0x40; +pub const diff_emph: u8 = 0x20; +/// A +Diff line added or removed whole, paired with no line it changed: +/// its row takes a faint tint, on its marker's byte. +pub const diff_whole: u8 = 0x10; +pub const syn_bits: u8 = 0x0f; + +/// A file section's hunks are parsed together, each side as one text, in +/// pieces: a piece takes whole hunks until it holds `diff_piece_lines` +/// lines, and a hunk longer than `diff_piece_cut` is cut there, counted +/// from the section's first hunk. So a view parses about what a file's +/// view does (its rows and a margin), however many small hunks it shows, +/// and a 20k-line new file is parsed where it is shown, not whole. A string +/// or comment across a cut colours as a fragment would. +const diff_piece_lines = 40; +const diff_piece_cut = 80; + +/// Painted hunk pieces, keyed by their bytes and language: scrolling back +/// over a hunk, or a terminal repainting the same output every frame, +/// parses nothing again. Freed by `stop`. +const DiffPiece = struct { hash: u64, styles: []u8, used: u64 }; +var diff_pieces: [96]?DiffPiece = @splat(null); +var diff_clock: u64 = 0; + +fn dropDiffPieces() void { + for (&diff_pieces) |*slot| { + if (slot.*) |piece| syntax_allocator.free(piece.styles); + slot.* = null; + } + diff_clock = 0; +} + +/// The styles of `content[start..end)` read as a unified diff: headers +/// muted, `@@` lines as keywords, and each hunk's code in the language of +/// the file its section names. A hunk's old side (context and removed +/// lines) and new side (context and added lines) are each parsed whole, so +/// a string or comment that spans lines colours as it does in the file. +/// Added and removed lines carry `diff_added`/`diff_removed`. A file in no +/// language pardes knows keeps the plain colours: added lines as strings, +/// removed ones as numbers. +pub fn highlightDiff(gpa: std.mem.Allocator, content: []const u8, start_raw: usize, end_raw: usize) ![]u8 { + return highlightDiffRows(gpa, content, start_raw, end_raw, false); +} + +/// The same for a terminal's rows, where a row the terminal wrapped +/// continues the diff line above it. +pub fn highlightDiffRows(gpa: std.mem.Allocator, content: []const u8, start_raw: usize, end_raw: usize, continued_rows: bool) ![]u8 { + const tz = tracy.zone(@src(), "highlightDiff"); + defer tz.end(); + const start_byte = @min(start_raw, content.len); + const end_byte = @max(start_byte, @min(end_raw, content.len)); + const styles = try gpa.alloc(u8, end_byte - start_byte); errdefer gpa.free(styles); @memset(styles, 0); + const Window = struct { + styles: []u8, + start: usize, + fn put(w: @This(), at: usize, bytes: []const u8) void { + const lo = @max(at, w.start); + const hi = @min(at + bytes.len, w.start + w.styles.len); + if (lo < hi) @memcpy(w.styles[lo - w.start .. hi - w.start], bytes[lo - at .. hi - at]); + } + fn fill(w: @This(), at: usize, len: usize, style: u8) void { + const lo = @max(at, w.start); + const hi = @min(at + len, w.start + w.styles.len); + if (lo < hi) @memset(w.styles[lo - w.start .. hi - w.start], style); + } + }; + const window: Window = .{ .styles = styles, .start = start_byte }; + + var walk: diff.Walk = .{ .continued_rows = continued_rows }; + // The run of hunk lines not painted yet, with each line's kind. + var piece_start: ?usize = null; + var piece_end: usize = 0; + var piece_path: []const u8 = ""; + var kinds: [diff_piece_cut]diff.Kind = undefined; + var nkinds: usize = 0; + var at = diff.anchorBefore(content, start_byte); + while (true) { + const nl = std.mem.indexOfScalarPos(u8, content, at, '\n') orelse content.len; + const got = walk.step(content[at..nl]); + const body = switch (got.kind) { + .context, .added, .removed, .no_newline => true, + // The next hunk of the section goes on in the same piece. + .hunk => piece_start != null and nkinds < diff_piece_lines, + else => false, + }; + if (piece_start) |ps| if (!body or nkinds == diff_piece_cut) { + if (piece_end >= start_byte) try paintDiffPiece(gpa, window, content[ps..piece_end], ps, kinds[0..nkinds], piece_path); + piece_start = null; + nkinds = 0; + }; + if (at >= end_byte and piece_start == null) break; + if (body and (piece_start != null or got.kind != .hunk)) { + if (piece_start == null) { + piece_start = at; + piece_path = walk.path(); + } + kinds[nkinds] = got.kind; + nkinds += 1; + piece_end = nl; + } else if (nl >= start_byte) { + const style: Syn = switch (got.kind) { + .hunk => .keyword, + .meta, .old_path, .new_path => .comment, + else => .none, + }; + if (style != .none) window.fill(at, nl - at, @backingInt(style)); + } + if (nl >= content.len) break; + at = nl + 1; + } + if (piece_start) |ps| if (piece_end >= start_byte) + try paintDiffPiece(gpa, window, content[ps..piece_end], ps, kinds[0..nkinds], piece_path); + return styles; +} +fn paintDiffPiece(gpa: std.mem.Allocator, window: anytype, bytes: []const u8, at: usize, kinds: []const diff.Kind, path: []const u8) !void { + const selected: ?Selected = if (enabled and syntax_started) (forExt(std.fs.path.extension(path)) catch null) else null; + if (selected) |sel| { + const hash = std.hash.Wyhash.hash(std.hash.Wyhash.hash(0, sel.name), bytes); + diff_clock += 1; + for (&diff_pieces) |*slot| if (slot.*) |*piece| if (piece.hash == hash and piece.styles.len == bytes.len) { + piece.used = diff_clock; + return window.put(at, piece.styles); + }; + const colored = try syntax_allocator.alloc(u8, bytes.len); + errdefer syntax_allocator.free(colored); + try paintHunk(gpa, colored, bytes, kinds, sel); + window.put(at, colored); + // An empty slot, else the least recently used. + var oldest: usize = 0; + for (&diff_pieces, 0..) |*slot, i| { + const piece = slot.* orelse { + oldest = i; + break; + }; + if (piece.used < diff_pieces[oldest].?.used) oldest = i; + } + if (diff_pieces[oldest]) |old| syntax_allocator.free(old.styles); + diff_pieces[oldest] = .{ .hash = hash, .styles = colored, .used = diff_clock }; + return; + } + // No language: each line in one colour, as diffs were always shown. + var lines = std.mem.splitScalar(u8, bytes, '\n'); var offset: usize = 0; - var lines = std.mem.splitScalar(u8, source, '\n'); - while (lines.next()) |line| { - const syn = diffLineSyn(line); - if (syn != .none) @memset(styles[offset .. offset + line.len], @intFromEnum(syn)); + for (kinds) |kind| { + const line = lines.next() orelse break; + const style: u8 = switch (kind) { + .added => @backingInt(Syn.string) | diff_added, + .removed => @backingInt(Syn.number) | diff_removed, + .no_newline => @backingInt(Syn.comment), + .hunk => @backingInt(Syn.keyword), + else => 0, + }; + if (style != 0) window.fill(at + offset, line.len, style); offset += line.len + 1; } - return styles; } -fn diffLineSyn(line: []const u8) Syn { - if (std.mem.startsWith(u8, line, "@@")) return .keyword; - if (std.mem.startsWith(u8, line, "+++") or - std.mem.startsWith(u8, line, "---") or - std.mem.startsWith(u8, line, "diff ") or - std.mem.startsWith(u8, line, "index ") or - std.mem.startsWith(u8, line, "\\ No newline")) return .comment; - if (line.len == 0) return .none; - if (line[0] == '+') return .string; - if (line[0] == '-') return .number; - return .none; +/// One piece's styles: its old side and its new side each parsed as one +/// text, and every line's code coloured from its side, behind its +/// one-character prefix. A `@@` line between its hunks is in neither. +fn paintHunk(gpa: std.mem.Allocator, out: []u8, bytes: []const u8, kinds: []const diff.Kind, sel: Selected) !void { + @memset(out, 0); + const old_src = try gpa.alloc(u8, bytes.len); + defer gpa.free(old_src); + const new_src = try gpa.alloc(u8, bytes.len); + defer gpa.free(new_src); + var old_len: usize = 0; + var new_len: usize = 0; + var lines = std.mem.splitScalar(u8, bytes, '\n'); + for (kinds) |kind| { + const line = lines.next() orelse break; + const code = line[codeStart(line, kind)..]; + // A row a terminal wrapped goes on the line above, as it was printed. + const joined = continuesRow(line, kind); + if (kind == .context or kind == .removed) { + if (joined and old_len > 0) old_len -= 1; + @memcpy(old_src[old_len..][0..code.len], code); + old_len += code.len; + old_src[old_len] = '\n'; + old_len += 1; + } + if (kind == .context or kind == .added) { + if (joined and new_len > 0) new_len -= 1; + @memcpy(new_src[new_len..][0..code.len], code); + new_len += code.len; + new_src[new_len] = '\n'; + new_len += 1; + } + } + // A side is parsed only when a line takes its colours from it: the old + // side for removed lines, the new side for added lines, and context + // from the new side, else from the old. A piece that only adds or only + // removes is one parse. + var removes = false; + var adds = false; + for (kinds) |kind| { + removes = removes or kind == .removed; + adds = adds or kind == .added; + } + const context_new = adds or !removes; + const old_colors = try gpa.alloc(u8, old_len); + defer gpa.free(old_colors); + @memset(old_colors, 0); + if (!context_new or removes) paint(old_colors, old_src[0..old_len], sel); + const new_colors = try gpa.alloc(u8, new_len); + defer gpa.free(new_colors); + @memset(new_colors, 0); + if (context_new) paint(new_colors, new_src[0..new_len], sel); + + lines = std.mem.splitScalar(u8, bytes, '\n'); + var offset: usize = 0; + var old_at: usize = 0; + var new_at: usize = 0; + for (kinds) |kind| { + const line = lines.next() orelse break; + defer offset += line.len + 1; + const skip = codeStart(line, kind); + const code_len = line.len - skip; + const dst = out[offset + skip ..][0..code_len]; + if (continuesRow(line, kind)) { + if ((kind == .context or kind == .removed) and old_at > 0) old_at -= 1; + if ((kind == .context or kind == .added) and new_at > 0) new_at -= 1; + } + switch (kind) { + .removed => @memcpy(dst, old_colors[old_at..][0..code_len]), + .added => @memcpy(dst, new_colors[new_at..][0..code_len]), + .context => @memcpy(dst, if (context_new) new_colors[new_at..][0..code_len] else old_colors[old_at..][0..code_len]), + .no_newline => @memset(out[offset..][0..line.len], @backingInt(Syn.comment)), + .hunk => @memset(out[offset..][0..line.len], @backingInt(Syn.keyword)), + else => {}, + } + if (kind == .context or kind == .removed) old_at += code_len + 1; + if (kind == .context or kind == .added) new_at += code_len + 1; + const flag: u8 = switch (kind) { + .added => diff_added, + .removed => diff_removed, + else => 0, + }; + if (flag != 0) for (out[offset..][0..line.len]) |*b| { + b.* |= flag; + }; + } +} + +fn continuesRow(line: []const u8, kind: diff.Kind) bool { + return line.len > 0 and codeStart(line, kind) == 0 and + (kind == .context or kind == .added or kind == .removed); +} + +/// Where a hunk line's code starts: past its prefix, which a blank context +/// line (its space stripped) and a terminal's wrapped row have none of. +fn codeStart(line: []const u8, kind: diff.Kind) usize { + if (line.len == 0) return 0; + return switch (kind) { + .context => @intFromBool(line[0] == ' '), + .added => @intFromBool(line[0] == '+'), + .removed => @intFromBool(line[0] == '-'), + else => 0, + }; } test "syntax a results row is coloured by the file its location names" { @@ -904,8 +1412,8 @@ test "syntax a results row is coloured by the file its location names" { try std.testing.expectEqual(content.len, styles.len); for (styles[0.."src/a.zig:1:1".len]) |b| try std.testing.expectEqual(@as(u8, 0), b); const zig_kw = std.mem.indexOf(u8, content, "struct").?; - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[zig_kw]); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[zig_kw + 5]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[zig_kw]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[zig_kw + 5]); const prose = std.mem.indexOf(u8, content, "just some prose").?; for (styles[prose .. prose + 14]) |b| try std.testing.expectEqual(@as(u8, 0), b); const bare = std.mem.indexOf(u8, content, "src/c.zig").?; @@ -933,25 +1441,11 @@ test "syntax codeAfterLocation takes whole-token locations and nothing else" { try std.testing.expect(codeAfterLocation("src/x.zig:7:2") == null); try std.testing.expect(codeAfterLocation("") == null); try std.testing.expect(codeAfterLocation(" leading space") == null); -} - -test "syntax tree-sitter allocator callbacks preserve and free exact allocations" { - syntax_allocator = std.testing.allocator; - defer syntax_allocator = undefined; - - var live: ?*anyopaque = syntaxCalloc(4, 1) orelse return error.OutOfMemory; - defer if (live) |pointer| syntaxFree(pointer); - const original: [*]u8 = @ptrCast(live.?); - try std.testing.expectEqualSlices(u8, &.{ 0, 0, 0, 0 }, original[0..4]); - @memcpy(original[0..4], "data"); - - live = syntaxRealloc(live, 32) orelse return error.OutOfMemory; - const grown: [*]u8 = @ptrCast(live.?); - try std.testing.expectEqualSlices(u8, "data", grown[0..4]); - - try std.testing.expect(syntaxCalloc(std.math.maxInt(usize), 2) == null); - try std.testing.expect(syntaxRealloc(live, 0) == null); - live = null; + const indented = " some directory/x.zig:7:2-9 return 42;"; + const nested = codeAfterLocation(indented) orelse return error.ShouldBeALocation; + try std.testing.expectEqualStrings("some directory/x.zig", nested.path); + try std.testing.expectEqualStrings(" return 42;", nested.text); + try std.testing.expectEqualStrings(nested.text, indented[nested.at..]); } test "syntax default full grammar set highlights Typst source" { @@ -967,14 +1461,14 @@ test "syntax default full grammar set highlights Typst source" { const keyword_at = std.mem.indexOf(u8, source, "let").?; const number_at = std.mem.indexOf(u8, source, "42").?; const string_at = std.mem.indexOf(u8, source, "\"hello\"").?; - try std.testing.expectEqual(Syn.comment, @as(Syn, @enumFromInt(styles[comment_at]))); - try std.testing.expectEqual(Syn.keyword, @as(Syn, @enumFromInt(styles[keyword_at]))); - try std.testing.expectEqual(Syn.number, @as(Syn, @enumFromInt(styles[number_at]))); - try std.testing.expectEqual(Syn.string, @as(Syn, @enumFromInt(styles[string_at]))); + try std.testing.expectEqual(Syn.comment, @as(Syn, @fromBackingInt(@intCast(styles[comment_at])))); + try std.testing.expectEqual(Syn.keyword, @as(Syn, @fromBackingInt(@intCast(styles[keyword_at])))); + try std.testing.expectEqual(Syn.number, @as(Syn, @fromBackingInt(@intCast(styles[number_at])))); + try std.testing.expectEqual(Syn.string, @as(Syn, @fromBackingInt(@intCast(styles[string_at])))); const short_ext = try highlightFileRange(std.testing.allocator, "paper.typ", source, 0, source.len); defer std.testing.allocator.free(short_ext); - try std.testing.expectEqual(Syn.keyword, @as(Syn, @enumFromInt(short_ext[keyword_at]))); + try std.testing.expectEqual(Syn.keyword, @as(Syn, @fromBackingInt(@intCast(short_ext[keyword_at])))); } test "syntax Typst markup constructs paint and raw blocks inject their language" { @@ -999,7 +1493,7 @@ test "syntax Typst markup constructs paint and raw blocks inject their language" const synAt = struct { fn f(s: []const u8, src: []const u8, needle: []const u8, offset: usize) Syn { - return @enumFromInt(s[std.mem.indexOf(u8, src, needle).? + offset]); + return @fromBackingInt(@intCast(s[std.mem.indexOf(u8, src, needle).? + offset])); } }.f; try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "= Heading", 0)); @@ -1039,7 +1533,7 @@ test "syntax markdown highlights markup, injects inline spans and fenced code bl const synAt = struct { fn f(s: []const u8, src: []const u8, needle: []const u8, offset: usize) Syn { - return @enumFromInt(s[std.mem.indexOf(u8, src, needle).? + offset]); + return @fromBackingInt(@intCast(s[std.mem.indexOf(u8, src, needle).? + offset])); } }.f; try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "Title", 0)); @@ -1067,8 +1561,8 @@ test "syntax inline fast path agrees with full Markdown query" { const actual = try std.testing.allocator.alloc(u8, source.len); defer std.testing.allocator.free(actual); for ([_]Syn{ .none, .keyword }) |background| { - @memset(expected, @intFromEnum(background)); - @memset(actual, @intFromEnum(background)); + @memset(expected, @backingInt(background)); + @memset(actual, @backingInt(background)); runQuery(expected, sel, tree, 0); inject(actual, source, tree.rootNode(), true); if (!std.mem.eql(u8, expected, actual)) std.debug.print("inline mismatch: {s}\n", .{source}); @@ -1107,40 +1601,224 @@ test "syntax plain Markdown keeps block styles without starting the inline parse const source = "# Heading\n\nPlain prose.\n\n indented code\n"; const styles = try highlightFileRange(std.testing.allocator, "a.md", source, 0, source.len); defer std.testing.allocator.free(styles); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[2]); - try std.testing.expectEqual(@intFromEnum(Syn.none), styles[std.mem.indexOf(u8, source, "Plain").?]); - try std.testing.expectEqual(@intFromEnum(Syn.string), styles[std.mem.indexOf(u8, source, "indented").?]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[2]); + try std.testing.expectEqual(@backingInt(Syn.none), styles[std.mem.indexOf(u8, source, "Plain").?]); + try std.testing.expectEqual(@backingInt(Syn.string), styles[std.mem.indexOf(u8, source, "indented").?]); for (specs) |spec| { if (std.mem.eql(u8, spec.name, "markdown_inline")) try std.testing.expect(spec.selected == null); } } test "syntax highlightDiff colors unified diff lines by prefix" { - const diff = + const diff_text = "diff --git a/x b/x\n" ++ "--- a/x\n" ++ "+++ b/x\n" ++ - "@@ -1,3 +1,3 @@\n" ++ + "@@ -1,2 +1,2 @@\n" ++ " context\n" ++ "-old line\n" ++ "+new line\n"; - const styles = try highlightDiff(std.testing.allocator, diff, 0, diff.len); + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "diff --git", 0)); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "--- a/x", 0)); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "+++ b/x", 0)); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "@@ -1,2", 0)); + try std.testing.expectEqual(Syn.none, diffSynAt(styles, diff_text, " context", 0)); + // A file in no known language: each line in one colour, and tinted. + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "-old line", 0)); + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "+new line", 0)); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "-old line").?] & diff_removed != 0); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "+new line").? + 4] & diff_added != 0); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, " context").?] & (diff_added | diff_removed) == 0); +} - const byteSyn = struct { - fn at(s: []const u8, src: []const u8, needle: []const u8) Syn { - const i = std.mem.indexOf(u8, src, needle).?; - return @enumFromInt(s[i]); - } - }.at; +fn diffSynAt(styles: []const u8, text: []const u8, needle: []const u8, offset: usize) Syn { + return @fromBackingInt(@intCast(styles[std.mem.indexOf(u8, text, needle).? + offset] & syn_bits)); +} - try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "diff --git")); - try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "--- a/x")); - try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "+++ b/x")); - try std.testing.expectEqual(Syn.keyword, byteSyn(styles, diff, "@@ -1,3")); - try std.testing.expectEqual(Syn.none, byteSyn(styles, diff, " context")); - try std.testing.expectEqual(Syn.number, byteSyn(styles, diff, "-old line")); - try std.testing.expectEqual(Syn.string, byteSyn(styles, diff, "+new line")); +test "syntax a Zig hunk is coloured as Zig, each side parsed whole" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + const diff_text = + "diff --git a/src/shape.zig b/src/shape.zig\n" ++ + "index 1111111..2222222 100644\n" ++ + "--- a/src/shape.zig\n" ++ + "+++ b/src/shape.zig\n" ++ + "@@ -1,6 +1,6 @@\n" ++ + " const Shape = struct {\n" ++ + "- side: u8 = 7,\n" ++ + "+ side: u16 = 42,\n" ++ + " const note =\n" ++ + "- \\\\old words defer\n" ++ + "+ \\\\new words defer\n" ++ + " ;\n" ++ + " };\n"; + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "struct", 0)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "7,", 0)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "42", 0)); + // A multiline string, told apart only by parsing the side it is on: + // `defer` inside it is no keyword. + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "old words defer", 10)); + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "new words defer", 10)); + // The prefix is no code: it carries the tint alone. + const added = std.mem.indexOf(u8, diff_text, "+ side").?; + try std.testing.expectEqual(diff_added, styles[added]); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "42").?] & diff_added != 0); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "7,").?] & diff_removed != 0); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "index 111", 0)); + + // A window in the middle is coloured as the whole is. + const from = std.mem.indexOf(u8, diff_text, " const note").?; + const part = try highlightDiff(std.testing.allocator, diff_text, from, diff_text.len); + defer std.testing.allocator.free(part); + try std.testing.expectEqualSlices(u8, styles[from..], part); +} + +test "syntax a Python hunk is coloured as Python" { + if (!enabled or !full_grammars) return; + start(std.testing.allocator); + defer stop(); + const diff_text = + "--- a/tool.py\n" ++ + "+++ b/tool.py\n" ++ + "@@ -1,3 +1,4 @@\n" ++ + " def run(n):\n" ++ + "- return n\n" ++ + "+ # doubled now\n" ++ + "+ return n * 2\n" ++ + " \n"; + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "def", 0)); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "return n\n", 0)); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "# doubled", 2)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "2\n", 0)); +} + +test "syntax a deleted file's lines are coloured in its language" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + const diff_text = + "diff --git a/gone.zig b/gone.zig\n" ++ + "deleted file mode 100644\n" ++ + "--- a/gone.zig\n" ++ + "+++ /dev/null\n" ++ + "@@ -1,2 +0,0 @@\n" ++ + "-const Gone = enum { a };\n" ++ + "-// the end\n"; + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "enum", 0)); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "// the end", 3)); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "enum").?] & diff_removed != 0); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "deleted file", 0)); +} + +test "syntax a hunk in no known language keeps the line colours" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + const diff_text = + "--- a/notes.unknownext\n" ++ + "+++ b/notes.unknownext\n" ++ + "@@ -1 +1 @@\n" ++ + "-const struct 1\n" ++ + "+const struct 2\n"; + const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "struct 1", 0)); + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "struct 2", 0)); + try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "+const struct 2", 0)); +} + +test "syntax git diff output in a command pane is told and coloured by its rows" { + if (!enabled) return; + start(std.testing.allocator); + defer stop(); + // A command pane's rows: its command line, then git's output, with one + // long added line the terminal wrapped onto a second row. + const rows = [_][]const u8{ + "git diff", + "diff --git a/main.zig b/main.zig", + "--- a/main.zig", + "+++ b/main.zig", + "@@ -1,2 +1,2 @@", + " pub fn main() void {", + "+ const answer: u32 = 42; // the ", + "wrapped rest", + "- return;", + "", + }; + try std.testing.expect(diff.looksLikeDiff(&rows)); + try std.testing.expect(!diff.looksLikeDiff(rows[5..])); + const text = try std.mem.join(std.testing.allocator, "\n", &rows); + defer std.testing.allocator.free(text); + const styles = try highlightDiffRows(std.testing.allocator, text, 0, text.len, true); + defer std.testing.allocator.free(styles); + try std.testing.expectEqual(Syn.none, diffSynAt(styles, text, "git diff", 0)); + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, text, "fn main", 0)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, text, "42", 0)); + // The wrapped row stays an added line, and inside the comment. + try std.testing.expect(styles[std.mem.indexOf(u8, text, "wrapped rest").?] & diff_added != 0); + try std.testing.expectEqual(Syn.comment, diffSynAt(styles, text, "wrapped rest", 0)); + try std.testing.expect(styles[std.mem.indexOf(u8, text, "return").?] & diff_removed != 0); +} + +test "syntax a section's small hunks share one parse" { + if (!enabled) return; + const gpa = std.testing.allocator; + start(gpa); + defer stop(); + const diff_text = + "--- a/x.zig\n+++ b/x.zig\n" ++ + "@@ -1 +1 @@\n-const a = 1;\n+const a = 2;\n" ++ + "@@ -10 +10 @@\n-const b = 1;\n+const b = 2;\n" ++ + "@@ -20 +20 @@\n-const c = 1;\n+const c = 2;\n" ++ + "--- a/y.zig\n+++ b/y.zig\n" ++ + "@@ -1 +1 @@\n-const d = 1;\n+const d = 2;\n"; + const styles = try highlightDiff(gpa, diff_text, 0, diff_text.len); + defer gpa.free(styles); + var cached: usize = 0; + for (diff_pieces) |slot| cached += @intFromBool(slot != null); + try std.testing.expectEqual(@as(usize, 2), cached); // x.zig's three hunks, and y.zig's + try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "@@ -10", 0)); + try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "2;\n@@ -20", 0)); + try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "@@ -10").?] & (diff_added | diff_removed) == 0); +} + +test "syntax a hunk painted once is kept, and a long one is parsed in pieces" { + if (!enabled) return; + const gpa = std.testing.allocator; + start(gpa); + defer stop(); + var text: std.Io.Writer.Allocating = .init(gpa); + defer text.deinit(); + const n = diff_piece_cut * 20 + 10; + try text.writer.print("--- /dev/null\n+++ b/big.zig\n@@ -0,0 +1,{d} @@\n", .{n}); + for (0..n) |i| try text.writer.print("+const v{d} = {d};\n", .{ i, i }); + const content = text.written(); + const at = std.mem.indexOf(u8, content, "+const v1000 ").?; + const window = try highlightDiff(gpa, content, at, at + 200); + defer gpa.free(window); + try std.testing.expectEqual(Syn.number, @as(Syn, @fromBackingInt(@intCast(window["+const v1000 = ".len] & syn_bits)))); + var cached: usize = 0; + for (diff_pieces) |slot| if (slot) |piece| { + cached += 1; + // One piece of many, not the whole hunk. + try std.testing.expect(piece.styles.len < content.len / 4); + }; + try std.testing.expectEqual(@as(usize, 1), cached); + const again = try highlightDiff(gpa, content, at, at + 200); + defer gpa.free(again); + try std.testing.expectEqualSlices(u8, window, again); + cached = 0; + for (diff_pieces) |slot| cached += @intFromBool(slot != null); + try std.testing.expectEqual(@as(usize, 1), cached); } test "syntax result fragments preserve source indentation and inline markup" { @@ -1149,6 +1827,7 @@ test "syntax result fragments preserve source indentation and inline markup" { defer stop(); const fixtures = [_]struct { path: []const u8, source: []const u8 }{ .{ .path = "a.zig", .source = " const number = 42; // note" }, + .{ .path = " nested path/a.zig", .source = " return call(42, \"hello\"); // note" }, .{ .path = "a.md", .source = "# Heading *slant*" }, .{ .path = "a.md", .source = "**bold** and `code`" }, .{ .path = "a.md", .source = " # this is indented code" }, @@ -1191,7 +1870,7 @@ test "syntax query filtering preserves upstream colors" { reference.query = raw_query; reference.capture_styles = try std.testing.allocator.alloc(u8, raw_query.captureCount()); defer std.testing.allocator.free(reference.capture_styles); - for (reference.capture_styles, 0..) |*style, id| style.* = @intFromEnum(synFor(raw_query.captureNameForId(@intCast(id)) orelse "")); + for (reference.capture_styles, 0..) |*style, id| style.* = @backingInt(synFor(raw_query.captureNameForId(@intCast(id)) orelse "")); const tree = selected.parser.parseString(source, null) orelse return error.ParseFailed; defer tree.destroy(); var expected: [source.len]u8 = @splat(0); @@ -1217,8 +1896,8 @@ test "syntax Zig keyword captures cover both bytes beyond line ten thousand" { defer std.testing.allocator.free(styles); for ([_][]const u8{ "fn", "if" }) |keyword| { const at = std.mem.indexOf(u8, code, keyword).?; - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[at]); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[at + 1]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[at]); + try std.testing.expectEqual(@backingInt(Syn.keyword), styles[at + 1]); } } @@ -1234,8 +1913,8 @@ test "syntax allocator switching clears default-runtime caches" { const restored = try highlightFileRange(std.testing.allocator, "a.zig", source, 0, source.len); defer std.testing.allocator.free(restored); defer stop(); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), restored[0]); - try std.testing.expectEqual(@intFromEnum(Syn.keyword), restored[1]); + try std.testing.expectEqual(@backingInt(Syn.keyword), restored[0]); + try std.testing.expectEqual(@backingInt(Syn.keyword), restored[1]); } test "syntax location context snapshots retain omitted multiline comment scope" { @@ -1259,8 +1938,8 @@ test "syntax location context snapshots retain omitted multiline comment scope" const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); for (styles[rows[0].code_start..][0..code.len]) |style| - try std.testing.expectEqual(@intFromEnum(Syn.comment), style); - try std.testing.expectEqual(@intFromEnum(Syn.none), styles[0]); + try std.testing.expectEqual(@backingInt(Syn.comment), style); + try std.testing.expectEqual(@backingInt(Syn.none), styles[0]); } test "syntax declaration locations clear snapshots without muting ordinary context" { @@ -1270,7 +1949,7 @@ test "syntax declaration locations clear snapshots without muting ordinary conte defer stop(); const header = "pub fn run() void {"; const content = "| \t" ++ header ++ "\n| \t const value = 42;\nfile.zig:3\t}\n"; - const snapshots = [_]u8{0xff} ** header.len; + const snapshots = @as([header.len]u8, @splat(0xff)); const Row = struct { path: []const u8 = "file.zig", code_start: usize, @@ -1286,7 +1965,7 @@ test "syntax declaration locations clear snapshots without muting ordinary conte const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); for (styles[3..][0..header.len]) |style| try std.testing.expectEqual(@as(u8, 0), style); - try std.testing.expectEqual(@intFromEnum(Syn.number), styles[std.mem.indexOf(u8, content, "42").?]); + try std.testing.expectEqual(@backingInt(Syn.number), styles[std.mem.indexOf(u8, content, "42").?]); } test "syntax stacked addresses do not split multiline source groups" { @@ -1306,11 +1985,11 @@ test "syntax stacked addresses do not split multiline source groups" { }; const styles = try highlightLocationRows(gpa, content, &rows); defer gpa.free(styles); - try std.testing.expectEqual(@intFromEnum(Syn.comment), styles[std.mem.indexOf(u8, content, "42").?]); - try std.testing.expectEqual(@intFromEnum(Syn.number), styles[std.mem.indexOf(u8, content, "7;").?]); + try std.testing.expectEqual(@backingInt(Syn.comment), styles[std.mem.indexOf(u8, content, "42").?]); + try std.testing.expectEqual(@backingInt(Syn.number), styles[std.mem.indexOf(u8, content, "7;").?]); for ([_][]const u8{ "a.c:1:1", "a.c:2:1", "a.c:3:1" }) |label| { const at = std.mem.indexOf(u8, content, label).?; - for (styles[at..][0..label.len]) |style| try std.testing.expectEqual(@intFromEnum(Syn.none), style); + for (styles[at..][0..label.len]) |style| try std.testing.expectEqual(@backingInt(Syn.none), style); } } @@ -1374,3 +2053,72 @@ test "syntax source analysis preserves declarations and injected colors" { try std.testing.expectEqual(@as(usize, 0), unsupported.declarations.len); try std.testing.expectEqual(@as(usize, 0), unsupported.colors.len); } + +test "tables are the pipe_tables tree-sitter finds" { + if (!enabled or !full_grammars) return; + start(std.testing.allocator); + defer stop(); + const Tables = @import("Tables.zig"); + const source = + "# Tables\n" ++ + "\n" ++ + "| a | b |\n" ++ + "|---|:-:|\n" ++ + "| 1 | 2 |\n" ++ + "no pipe row\n" ++ + "\n" ++ + "a | b\n" ++ + "--|--\n" ++ + "c | d\n" ++ + "# heading ends it\n" ++ + "| x |\n" ++ + "|---|\n" ++ + "\n" ++ + "```\n" ++ + "| in | fence |\n" ++ + "|----|-------|\n" ++ + "```\n" ++ + "\n" ++ + "| one | two |\n" ++ + "|-----|\n" ++ + "\n" ++ + " | indented | table |\n" ++ + " | -------- | ----- |\n" ++ + " | row | here |\n" ++ + "> quote ends\n" ++ + "\n" ++ + "| esc \\| aped | two |\n" ++ + "| --- | --- |\n" ++ + "- list ends\n"; + const selected = (try forLang("markdown")).?; + const tree = selected.parser.parseString(source, null) orelse return error.ParseFailed; + defer tree.destroy(); + // Every pipe_table node's lines, wherever it is in the tree. + var found: std.ArrayList([2]u32) = .empty; + defer found.deinit(std.testing.allocator); + var stack: std.ArrayList(ts.Node) = .empty; + defer stack.deinit(std.testing.allocator); + try stack.append(std.testing.allocator, tree.rootNode()); + while (stack.pop()) |node| { + if (std.mem.eql(u8, node.kind(), "pipe_table")) { + const end = node.endPoint(); + try found.append(std.testing.allocator, .{ node.startPoint().row, end.row + @intFromBool(end.column > 0) }); + continue; + } + var i = node.childCount(); + while (i > 0) : (i -= 1) if (node.child(i - 1)) |child| try stack.append(std.testing.allocator, child); + } + // The tables, the indented one and the escaped pipe's; not the fence's + // or the one whose delimiter has a cell too few. + try std.testing.expectEqual(@as(usize, 5), found.items.len); + var t: Tables = .{}; + defer t.deinit(std.testing.allocator); + try t.scan(std.testing.allocator, source); + var mine: std.ArrayList([2]u32) = .empty; + defer mine.deinit(std.testing.allocator); + for (t.list.items) |table| try mine.append(std.testing.allocator, .{ table.first, table.end }); + std.testing.expectEqualSlices([2]u32, found.items, mine.items) catch |e| { + std.debug.print("tree-sitter {any}\nTables {any}\n", .{ found.items, mine.items }); + return e; + }; +} |
