summaryrefslogtreecommitdiff
path: root/src/syntax.zig
diff options
context:
space:
mode:
Diffstat (limited to 'src/syntax.zig')
-rw-r--r--src/syntax.zig1090
1 files changed, 919 insertions, 171 deletions
diff --git a/src/syntax.zig b/src/syntax.zig
index e637bf67..985a9888 100644
--- a/src/syntax.zig
+++ b/src/syntax.zig
@@ -3,6 +3,13 @@ const config = @import("pardes_config");
const tracy = @import("tracy.zig");
const grammar_manifest = @import("grammar_manifest.zig");
const look = @import("look.zig");
+const regexp = @import("regexp.zig");
+const c_heap = @import("c_heap");
+const diff = @import("diff.zig");
+
+test {
+ _ = diff;
+}
pub const enabled = config.syntax_highlighting;
const zig_grammar = config.syntax_zig_grammar;
@@ -26,6 +33,10 @@ const Spec = struct {
exts: []const []const u8,
language: *const fn () callconv(.c) *const ts.Language,
query_src: []const u8,
+ /// helix's textobjects.scm for the grammar (vendor/queries), "" for none
+ objects_src: []const u8 = "",
+ objects: ?*ts.Query = null,
+ objects_failed: bool = false,
selected: ?Selected = null,
capture_styles: [256]u8 = undefined,
};
@@ -130,6 +141,7 @@ fn initSpecs() [specCount()]Spec {
.exts = g.exts,
.language = @extern(LanguageFn, .{ .name = "tree_sitter_" ++ g.name }),
.query_src = querySrc(g),
+ .objects_src = @field(ts_queries, g.name ++ "_textobjects"),
};
i += 1;
}
@@ -139,8 +151,11 @@ fn initSpecs() [specCount()]Spec {
var specs = initSpecs();
-const allocation_header_size = 16;
+/// What tree-sitter allocates from, the runtime and every grammar's external
+/// scanner (build.zig compiles them to use the runtime's allocator):
+/// `start`'s allocator, through the shared C heap.
var syntax_allocator: std.mem.Allocator = undefined;
+const syntax_heap = c_heap.Heap(&syntax_allocator);
var syntax_started = false;
extern fn ts_set_allocator(
@@ -150,56 +165,13 @@ extern fn ts_set_allocator(
new_free: ?*const fn (ptr: ?*anyopaque) callconv(.c) void,
) void;
-fn syntaxAlloc(size_arg: usize) callconv(.c) ?*anyopaque {
- const size = @max(size_arg, 1);
- const total = std.math.add(usize, allocation_header_size, size) catch return null;
- const bytes = syntax_allocator.alignedAlloc(u8, .@"16", total) catch return null;
- const header: *align(16) usize = @ptrCast(bytes.ptr);
- header.* = total;
- return @ptrCast(bytes.ptr + allocation_header_size);
-}
-
-fn syntaxCalloc(count: usize, size: usize) callconv(.c) ?*anyopaque {
- const len = std.math.mul(usize, count, size) catch return null;
- const pointer = syntaxAlloc(len) orelse return null;
- const bytes: [*]u8 = @ptrCast(pointer);
- @memset(bytes[0..len], 0);
- return pointer;
-}
-
-fn syntaxRealloc(ptr: ?*anyopaque, new_size: usize) callconv(.c) ?*anyopaque {
- const pointer = ptr orelse return syntaxAlloc(new_size);
- if (new_size == 0) {
- syntaxFree(pointer);
- return null;
- }
- const user: [*]u8 = @ptrCast(pointer);
- const header: *align(16) usize = @ptrCast(@alignCast(user - allocation_header_size));
- const old_total = header.*;
- const old_bytes: []align(16) u8 = @as([*]align(16) u8, @ptrCast(header))[0..old_total];
- const new_total = std.math.add(usize, allocation_header_size, new_size) catch return null;
- const new_bytes = syntax_allocator.realloc(old_bytes, new_total) catch return null;
- const new_header: *align(16) usize = @ptrCast(new_bytes.ptr);
- new_header.* = new_total;
- return @ptrCast(new_bytes.ptr + allocation_header_size);
-}
-
-fn syntaxFree(ptr: ?*anyopaque) callconv(.c) void {
- const pointer = ptr orelse return;
- const user: [*]u8 = @ptrCast(pointer);
- const header: *align(16) usize = @ptrCast(@alignCast(user - allocation_header_size));
- const total = header.*;
- const bytes: []align(16) u8 = @as([*]align(16) u8, @ptrCast(header))[0..total];
- syntax_allocator.free(bytes);
-}
-
pub fn start(gpa: std.mem.Allocator) void {
if (comptime enabled) {
std.debug.assert(!syntax_started);
stop();
syntax_allocator = gpa;
syntax_started = true;
- ts_set_allocator(syntaxAlloc, syntaxCalloc, syntaxRealloc, syntaxFree);
+ ts_set_allocator(syntax_heap.malloc, syntax_heap.calloc, syntax_heap.realloc, syntax_heap.free);
}
}
@@ -212,7 +184,11 @@ pub fn stop() void {
selected.query.destroy();
}
spec.selected = null;
+ if (spec.objects) |q| q.destroy();
+ spec.objects = null;
+ spec.objects_failed = false;
}
+ dropDiffPieces();
ts_set_allocator(null, null, null, null);
syntax_started = false;
syntax_allocator = undefined;
@@ -238,7 +214,7 @@ fn ensure(spec: *Spec) !Selected {
const capture_styles = spec.capture_styles[0..query.captureCount()];
for (capture_styles, 0..) |*style, id| {
const name = query.captureNameForId(@intCast(id)) orelse "";
- style.* = @intFromEnum(synFor(name));
+ style.* = @backingInt(synFor(name));
if (style.* == 0) query.disableCapture(name);
}
const parser = ts.Parser.create();
@@ -265,35 +241,8 @@ fn forExt(ext: []const u8) !?Selected {
return null;
}
-const LangAlias = struct { []const u8, []const u8 };
-const lang_aliases = [_]LangAlias{
- .{ "js", "javascript" },
- .{ "jsx", "javascript" },
- .{ "mjs", "javascript" },
- .{ "py", "python" },
- .{ "sh", "bash" },
- .{ "shell", "bash" },
- .{ "zsh", "bash" },
- .{ "bash", "bash" },
- .{ "rs", "rust" },
- .{ "c++", "cpp" },
- .{ "cxx", "cpp" },
- .{ "cc", "cpp" },
- .{ "cs", "c_sharp" },
- .{ "kt", "kotlin" },
- .{ "rb", "ruby" },
- .{ "ml", "ocaml" },
- .{ "hs", "haskell" },
-};
-
fn forLang(name: []const u8) !?Selected {
- var canonical = name;
- for (lang_aliases) |a| {
- if (std.ascii.eqlIgnoreCase(name, a[0])) {
- canonical = a[1];
- break;
- }
- }
+ const canonical = if (grammar_manifest.byName(name)) |i| grammar_manifest.all[i].name else name;
for (&specs) |*spec| {
if (std.ascii.eqlIgnoreCase(canonical, spec.name)) return try ensure(spec);
}
@@ -355,6 +304,56 @@ pub fn highlightFileRange(gpa: std.mem.Allocator, path: []const u8, content: []c
return styles;
}
+/// Whether `highlightApart` can colour `path` on a worker: a grammar with
+/// no injections (markdown's and typst's reach the shared parsers), made
+/// ready here, on the loop, so the worker only reads its query.
+pub fn readyApart(path: []const u8) bool {
+ if (!enabled) return false;
+ const selected = (forExt(std.fs.path.extension(path)) catch return false) orelse return false;
+ return !std.mem.eql(u8, selected.name, "markdown") and !std.mem.eql(u8, selected.name, "typst");
+}
+
+/// `highlightFileRange` of `source` (a copy of the range), on a worker: a
+/// parser and a query cursor of its own, the grammar's query shared (it
+/// is only read); `halt` ends the parse early. After `readyApart`.
+pub fn highlightApart(gpa: std.mem.Allocator, path: []const u8, source: []const u8, halt: *const std.atomic.Value(bool)) ![]u8 {
+ if (comptime !enabled) return &.{};
+ const shared = (try forExt(std.fs.path.extension(path))) orelse return &.{};
+ const parser = ts.Parser.create();
+ defer parser.destroy();
+ try parser.setLanguage(shared.lang);
+ const cursor = ts.QueryCursor.create();
+ defer cursor.destroy();
+ const Reader = struct {
+ text: []const u8,
+ fn read(payload: ?*anyopaque, byte: u32, _: ts.Point, n: *u32) callconv(.c) [*c]const u8 {
+ const r: *@This() = @ptrCast(@alignCast(payload.?));
+ const at = @min(byte, r.text.len);
+ n.* = @intCast(r.text.len - at);
+ return r.text.ptr + at;
+ }
+ fn progress(state: *ts.Parser.State) callconv(.c) bool {
+ const flag: *const std.atomic.Value(bool) = @ptrCast(@alignCast(state.payload.?));
+ return flag.load(.monotonic);
+ }
+ };
+ var reader: Reader = .{ .text = source };
+ const tree = parser.parseWithOptions(
+ .{ .payload = &reader, .read = Reader.read, .encoding = .utf8 },
+ null,
+ .{ .payload = @constCast(halt), .progress_callback = @ptrCast(&Reader.progress) },
+ ) orelse return error.Canceled;
+ defer tree.destroy();
+ if (halt.load(.monotonic)) return error.Canceled;
+ const styles = try gpa.alloc(u8, source.len);
+ @memset(styles, 0);
+ var own = shared;
+ own.parser = parser;
+ own.cursor = cursor;
+ runQuery(styles, own, tree, 0);
+ return styles;
+}
+
/// Source spans are independent of editor rows. Locations output uses the same
/// declarations as sticky editor headers. Byte ends are exclusive; rows are
/// zero-based and inclusive, including the last signature/opening-brace row.
@@ -376,6 +375,233 @@ pub fn supportsPath(path: []const u8) bool {
return false;
}
+/// A whole file's parse, kept by the file for the keys that walk its nodes
+/// (helix's Alt-o and kin). Opaque outside this file; null without a grammar.
+pub fn parseTree(path: []const u8, content: []const u8) ?*anyopaque {
+ if (!enabled) return null;
+ const selected = (forExt(std.fs.path.extension(path)) catch return null) orelse return null;
+ const tree = selected.parser.parseString(content, null) orelse return null;
+ return @ptrCast(tree);
+}
+
+pub fn freeTree(tree: *anyopaque) void {
+ if (!enabled) return;
+ const t: *ts.Tree = @ptrCast(@alignCast(tree));
+ t.destroy();
+}
+
+/// Where a node walk goes from a range (helix object.rs and movement.rs).
+pub const NodeWalk = @import("modal.zig").Normal.NodeWalk;
+pub const Span = struct { from: usize, to: usize };
+
+/// The spans `walk` takes the byte range [from, to) to, into `out`; `at` is
+/// the range's cursor. None when the tree has nothing to say there.
+pub fn walkNodes(tree_ptr: *anyopaque, walk: NodeWalk, from: usize, to: usize, at: usize, out: []Span) usize {
+ if (!enabled or out.len == 0) return 0;
+ const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr));
+ const lo: u32 = @intCast(@min(from, std.math.maxInt(u32)));
+ const hi: u32 = @intCast(@min(to, std.math.maxInt(u32)));
+ if (walk == .parent_end or walk == .parent_start) {
+ var node = tree.rootNode().namedDescendantForByteRange(lo, hi) orelse return 0;
+ if (walk == .parent_end) {
+ out[0] = .{ .from = node.endByte(), .to = node.endByte() };
+ return 1;
+ }
+ // already at the node's lo: its first ancestor that starts earlier
+ if (node.startByte() == at) {
+ const first = node.startByte();
+ while (node.startByte() >= first or !node.isNamed()) node = node.parent() orelse break;
+ }
+ out[0] = .{ .from = node.startByte(), .to = node.startByte() };
+ return 1;
+ }
+ var cursor = tree.walk();
+ defer cursor.destroy();
+ // helix TreeCursor.reset_to_byte_range: the smallest node holding the range
+ while (true) {
+ const node = cursor.node();
+ if (lo < node.startByte() or hi > node.endByte()) {
+ _ = cursor.gotoParent();
+ break;
+ }
+ if (cursor.gotoFirstChildForByte(lo) == null) break;
+ }
+ switch (walk) {
+ .expand => while (cursor.node().startByte() == lo and cursor.node().endByte() == hi) {
+ if (!cursor.gotoParent()) break;
+ },
+ .shrink => _ = cursor.gotoFirstChild(),
+ .next_sibling => while (!cursor.gotoNextSibling()) {
+ if (!cursor.gotoParent()) break;
+ },
+ .prev_sibling => while (!cursor.gotoPreviousSibling()) {
+ if (!cursor.gotoParent()) break;
+ },
+ .all_siblings, .all_children => {
+ if (walk == .all_siblings) {
+ while (true) {
+ if (!cursor.gotoParent()) return 0;
+ if (cursor.node().childCount() > 1) break;
+ }
+ }
+ // the named children, or nothing to say
+ var n: usize = 0;
+ if (!cursor.gotoFirstChild()) return 0;
+ while (true) {
+ const child = cursor.node();
+ if (child.isNamed() and n < out.len) {
+ out[n] = .{ .from = child.startByte(), .to = child.endByte() };
+ n += 1;
+ }
+ if (!cursor.gotoNextSibling()) break;
+ }
+ return n;
+ },
+ .parent_end, .parent_start => unreachable,
+ }
+ const node = cursor.node();
+ out[0] = .{ .from = node.startByte(), .to = node.endByte() };
+ return 1;
+}
+
+/// helix's textobjects, which its `]f`, `mif` and kin select: the name a
+/// query's captures begin with (`function.around`, `test.inside`).
+pub const Object = @import("modal.zig").Normal.Object;
+
+fn objectName(o: Object) []const u8 {
+ return if (o == .xml_element) "xml-element" else @tagName(o);
+}
+
+/// The grammar's textobject query for `path`, compiled the first time; null
+/// when the grammar has none.
+fn objectQuery(path: []const u8) ?*ts.Query {
+ const ext = std.fs.path.extension(path);
+ for (&specs) |*spec| for (spec.exts) |choice| {
+ if (!std.ascii.eqlIgnoreCase(ext, choice)) continue;
+ if (spec.objects) |q| return q;
+ if (spec.objects_src.len == 0 or spec.objects_failed) return null;
+ var error_offset: u32 = 0;
+ spec.objects = ts.Query.create(spec.language(), spec.objects_src, &error_offset) catch {
+ spec.objects_failed = true;
+ return null;
+ };
+ return spec.objects;
+ };
+ return null;
+}
+
+fn captureId(q: *const ts.Query, name: []const u8) ?u32 {
+ for (0..q.captureCount()) |i| if (std.mem.eql(u8, q.captureNameForId(@intCast(i)) orelse "", name)) return @intCast(i);
+ return null;
+}
+
+/// A match's `capture`, as one span over all its nodes (helix's grouped
+/// capture), when the match's `#eq?` and `#match?` hold.
+fn matchSpan(q: *const ts.Query, match: anytype, capture: u32, text: []const u8) ?Span {
+ var span: ?Span = null;
+ for (match.captures) |c| if (c.index == capture) {
+ const s: Span = .{ .from = c.node.startByte(), .to = c.node.endByte() };
+ span = if (span) |old| .{ .from = @min(old.from, s.from), .to = @max(old.to, s.to) } else s;
+ };
+ var found = span orelse return null;
+ // a node that takes its line's newline (this zig grammar's comments)
+ // is the line without it, as helix's grammars have it
+ if (found.to > found.from + 1 and found.to <= text.len and text[found.to - 1] == '\n') found.to -= 1;
+ const steps = q.predicatesForPattern(match.pattern_index);
+ var i: usize = 0;
+ while (i < steps.len) {
+ var end = i;
+ while (end < steps.len and steps[end].type != .done) end += 1;
+ const pred = steps[i..end];
+ i = end + 1;
+ if (pred.len != 3 or pred[0].type != .string or pred[1].type != .capture) continue;
+ const op = q.stringValueForId(pred[0].value_id) orelse continue;
+ const subject = for (match.captures) |c| {
+ if (c.index == pred[1].value_id) break text[@min(c.node.startByte(), text.len)..@min(c.node.endByte(), text.len)];
+ } else continue;
+ const want = if (pred[2].type == .string) q.stringValueForId(pred[2].value_id) orelse continue else for (match.captures) |c| {
+ if (c.index == pred[2].value_id) break text[@min(c.node.startByte(), text.len)..@min(c.node.endByte(), text.len)];
+ } else continue;
+ if (std.mem.eql(u8, op, "eq?")) {
+ if (!std.mem.eql(u8, subject, want)) return null;
+ } else if (std.mem.eql(u8, op, "match?")) {
+ var re = regexp.Regex.compile(want) catch continue;
+ if ((re.find(subject, 0, subject.len, subject.len) catch null) == null) return null;
+ }
+ }
+ return found;
+}
+
+/// `mi`/`ma`: the smallest `object.inside` (or `.around`) holding byte `at`
+/// (helix textobject_treesitter). None without a query or such an object.
+pub fn objectAt(path: []const u8, tree_ptr: *anyopaque, text: []const u8, object: Object, around: bool, at: usize) ?Span {
+ if (!enabled) return null;
+ const q = objectQuery(path) orelse return null;
+ var buf: [32]u8 = undefined;
+ const want = std.fmt.bufPrint(&buf, "{s}.{s}", .{ objectName(object), if (around) "around" else "inside" }) catch return null;
+ const capture = captureId(q, want) orelse return null;
+ const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr));
+ const cursor = ts.QueryCursor.create();
+ defer cursor.destroy();
+ cursor.exec(q, tree.rootNode());
+ var best: ?Span = null;
+ while (cursor.nextMatch()) |match| {
+ const s = matchSpan(q, match, capture, text) orelse continue;
+ if (!(s.from <= at and at < s.to)) continue;
+ if (best == null or s.to - s.from < best.?.to - best.?.from) best = s;
+ }
+ const s = best orelse return null;
+ if (s.from >= text.len or s.to >= text.len) return null;
+ return s;
+}
+
+/// `]f`/`[f` and kin: the next object starting after byte `at`, or the
+/// previous one ending before it (helix goto_treesitter_object), from the
+/// first of `.movement`, `.around`, `.inside` the query names.
+pub fn objectNext(path: []const u8, tree_ptr: *anyopaque, text: []const u8, object: Object, forward: bool, at: usize) ?Span {
+ if (!enabled) return null;
+ const q = objectQuery(path) orelse return null;
+ var buf: [32]u8 = undefined;
+ const capture = for ([_][]const u8{ "movement", "around", "inside" }) |kind| {
+ const want = std.fmt.bufPrint(&buf, "{s}.{s}", .{ objectName(object), kind }) catch return null;
+ if (captureId(q, want)) |id| break id;
+ } else return null;
+ const tree: *const ts.Tree = @ptrCast(@alignCast(tree_ptr));
+ const cursor = ts.QueryCursor.create();
+ defer cursor.destroy();
+ cursor.exec(q, tree.rootNode());
+ var best: ?Span = null;
+ while (cursor.nextMatch()) |match| {
+ const s = matchSpan(q, match, capture, text) orelse continue;
+ if (forward) {
+ if (s.from <= at) continue;
+ if (best == null or s.from < best.?.from or (s.from == best.?.from and s.to > best.?.to)) best = s;
+ } else {
+ if (s.to >= at) continue;
+ if (best == null or s.to > best.?.to or (s.to == best.?.to and s.from < best.?.from)) best = s;
+ }
+ }
+ const s = best orelse return null;
+ if (s.from >= text.len or s.to >= text.len) return null;
+ return s;
+}
+
+test "every vendored textobject query compiles against its grammar" {
+ if (!enabled) return;
+ var failed: usize = 0;
+ for (&specs) |*spec| {
+ if (spec.objects_src.len == 0) continue;
+ var error_offset: u32 = 0;
+ const q = ts.Query.create(spec.language(), spec.objects_src, &error_offset) catch {
+ std.debug.print("textobjects for {s} do not compile (offset {d})\n", .{ spec.name, error_offset });
+ failed += 1;
+ continue;
+ };
+ q.destroy();
+ }
+ try std.testing.expectEqual(@as(usize, 0), failed);
+}
+
/// Owned source analysis. Both slices use the allocator passed to analyzeSource.
/// The parsed tree is released before returning; callers can cache this result.
pub const SourceAnalysis = struct {
@@ -396,7 +622,7 @@ pub fn analyzeSource(gpa: std.mem.Allocator, path: []const u8, content: []const
defer tree.destroy();
var result: SourceAnalysis = .{};
errdefer result.deinit(gpa);
- if (want_declarations) result.declarations = try declarationsFromTree(gpa, content, tree);
+ if (want_declarations) result.declarations = try declarationsFromTree(gpa, content, tree, null);
if (want_colors) {
result.colors = try gpa.alloc(u8, content.len);
@memset(result.colors, 0);
@@ -410,14 +636,55 @@ pub fn contextDeclarations(gpa: std.mem.Allocator, path: []const u8, content: []
return analysis.declarations;
}
-fn declarationsFromTree(gpa: std.mem.Allocator, content: []const u8, tree: *ts.Tree) ![]ContextDeclaration {
+/// The declarations of `content` (its path's language), parsed with a
+/// parser of its own, so a worker thread may ask: the loop's parsers
+/// (`ensure`'s, one a language) are not touched, and a language is safe to
+/// share. `stop`, set from another thread, ends the parse and the walk
+/// early (error.Canceled): a 1.2 MB file is seconds in a Debug build.
+pub fn declarationsApart(gpa: std.mem.Allocator, path: []const u8, content: []const u8, halt: *const std.atomic.Value(bool)) ![]ContextDeclaration {
+ if (comptime !enabled) return &.{};
+ if (content.len == 0) return &.{};
+ const ext = std.fs.path.extension(path);
+ const spec = specs: for (&specs) |*spec| {
+ for (spec.exts) |choice| if (std.ascii.eqlIgnoreCase(ext, choice)) break :specs spec;
+ } else return &.{};
+ const parser = ts.Parser.create();
+ defer parser.destroy();
+ try parser.setLanguage(spec.language());
+ const Reader = struct {
+ text: []const u8,
+ fn read(payload: ?*anyopaque, byte: u32, _: ts.Point, n: *u32) callconv(.c) [*c]const u8 {
+ const r: *@This() = @ptrCast(@alignCast(payload.?));
+ const at = @min(byte, r.text.len);
+ n.* = @intCast(r.text.len - at);
+ return r.text.ptr + at;
+ }
+ fn progress(state: *ts.Parser.State) callconv(.c) bool {
+ const flag: *const std.atomic.Value(bool) = @ptrCast(@alignCast(state.payload.?));
+ return flag.load(.monotonic);
+ }
+ };
+ var reader: Reader = .{ .text = content };
+ const tree = parser.parseWithOptions(
+ .{ .payload = &reader, .read = Reader.read, .encoding = .utf8 },
+ null,
+ .{ .payload = @constCast(halt), .progress_callback = @ptrCast(&Reader.progress) },
+ ) orelse return error.Canceled;
+ defer tree.destroy();
+ return declarationsFromTree(gpa, content, tree, halt);
+}
+
+fn declarationsFromTree(gpa: std.mem.Allocator, content: []const u8, tree: *ts.Tree, halt: ?*const std.atomic.Value(bool)) ![]ContextDeclaration {
var result: std.ArrayList(ContextDeclaration) = .empty;
errdefer result.deinit(gpa);
// The cursor retains its ancestor stack: sibling visits do not repeatedly
// reconstruct parents in broad syntax trees. Preorder preserves source order.
var cursor = tree.rootNode().walk();
defer cursor.destroy();
+ var visited: usize = 0;
while (true) {
+ visited += 1;
+ if (halt) |flag| if (visited % 4096 == 0 and flag.load(.monotonic)) return error.Canceled;
const node = cursor.node();
if (node.isNamed()) if (contextSpan(node, content)) |span| {
if (result.items.len == 0 or result.items[result.items.len - 1].start_line != span.start_line or
@@ -677,7 +944,13 @@ pub fn highlightLocationRows(gpa: std.mem.Allocator, content: []const u8, rows:
if (comptime @hasField(@TypeOf(row), "location_end")) {
address_only = row.location_end > 0 and source_start == end;
}
- if (!address_only) {
+ // A row its producer coloured whole is coloured below: no
+ // fragment of it to parse.
+ var coloured = false;
+ if (comptime @hasField(@TypeOf(row), "colors")) {
+ coloured = row.colors.len > 0 and row.colors.len >= end - source_start;
+ }
+ if (!address_only and !coloured) {
sources[source_count] = .{ .index = index, .start = source_start, .len = end - source_start };
source_count += 1;
}
@@ -757,13 +1030,10 @@ pub fn highlightLocationRows(gpa: std.mem.Allocator, content: []const u8, rows:
}
fn codeAfterLocation(line: []const u8) ?struct { path: []const u8, at: usize, text: []const u8 } {
- const token_end = std.mem.indexOfAny(u8, line, " \t") orelse return null;
- if (token_end == 0) return null;
- const token = line[0..token_end];
- const target = look.parsePathLine(token);
- if (target.end != token.len) return null;
+ const trimmed = std.mem.trimStart(u8, line, " \t");
+ const target = @import("locations.zig").parseRaw(trimmed);
if (target.at.line == 0) return null;
- const at = token_end + 1;
+ const at = line.len - trimmed.len + target.code_start;
if (at >= line.len) return null;
return .{ .path = target.path, .at = at, .text = line[at..] };
}
@@ -786,11 +1056,11 @@ test "syntax formatted locations color hidden context and preserve multiline sou
const styles = try highlightLocationRows(gpa, content, &rows);
defer gpa.free(styles);
const body = std.mem.indexOf(u8, content, "comment body").?;
- try std.testing.expectEqual(@intFromEnum(Syn.comment), styles[body]);
- try std.testing.expectEqual(@intFromEnum(Syn.none), styles[body - 1]);
+ try std.testing.expectEqual(@backingInt(Syn.comment), styles[body]);
+ try std.testing.expectEqual(@backingInt(Syn.none), styles[body - 1]);
const number = std.mem.indexOf(u8, content, "42").?;
- try std.testing.expectEqual(@intFromEnum(Syn.number), styles[number]);
- try std.testing.expectEqual(@intFromEnum(Syn.none), styles[0]);
+ try std.testing.expectEqual(@backingInt(Syn.number), styles[number]);
+ try std.testing.expectEqual(@backingInt(Syn.none), styles[0]);
}
test "syntax formatted locations separate alignment from Markdown indentation" {
@@ -806,8 +1076,8 @@ test "syntax formatted locations separate alignment from Markdown indentation" {
};
const styles = try highlightLocationRows(gpa, content, &rows);
defer gpa.free(styles);
- try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[std.mem.indexOf(u8, content, "Heading").?]);
- try std.testing.expectEqual(@intFromEnum(Syn.string), styles[std.mem.indexOf(u8, content, "Code").?]);
+ try std.testing.expectEqual(@backingInt(Syn.keyword), styles[std.mem.indexOf(u8, content, "Heading").?]);
+ try std.testing.expectEqual(@backingInt(Syn.string), styles[std.mem.indexOf(u8, content, "Code").?]);
}
fn inject(styles: []u8, source: []const u8, node: ts.Node, markdown: bool) void {
const kind = node.kind();
@@ -858,35 +1128,273 @@ fn childOfKind(node: ts.Node, kind: []const u8) ?ts.Node {
return null;
}
-pub fn highlightDiff(gpa: std.mem.Allocator, content: []const u8, start_byte_raw: usize, end_byte_raw: usize) ![]u8 {
- const start_byte = @min(start_byte_raw, content.len);
- const end_byte = @max(start_byte, @min(end_byte_raw, content.len));
- const source = content[start_byte..end_byte];
- const styles = try gpa.alloc(u8, source.len);
+/// A diff line's style byte carries whether the line was added or removed
+/// over its syntax style (the low bits), so the painter can tint its row
+/// and mark its prefix; Diff marks the words a changed line changed
+/// (hunks.zig), drawn on a stronger tint.
+pub const diff_added: u8 = 0x80;
+pub const diff_removed: u8 = 0x40;
+pub const diff_emph: u8 = 0x20;
+/// A +Diff line added or removed whole, paired with no line it changed:
+/// its row takes a faint tint, on its marker's byte.
+pub const diff_whole: u8 = 0x10;
+pub const syn_bits: u8 = 0x0f;
+
+/// A file section's hunks are parsed together, each side as one text, in
+/// pieces: a piece takes whole hunks until it holds `diff_piece_lines`
+/// lines, and a hunk longer than `diff_piece_cut` is cut there, counted
+/// from the section's first hunk. So a view parses about what a file's
+/// view does (its rows and a margin), however many small hunks it shows,
+/// and a 20k-line new file is parsed where it is shown, not whole. A string
+/// or comment across a cut colours as a fragment would.
+const diff_piece_lines = 40;
+const diff_piece_cut = 80;
+
+/// Painted hunk pieces, keyed by their bytes and language: scrolling back
+/// over a hunk, or a terminal repainting the same output every frame,
+/// parses nothing again. Freed by `stop`.
+const DiffPiece = struct { hash: u64, styles: []u8, used: u64 };
+var diff_pieces: [96]?DiffPiece = @splat(null);
+var diff_clock: u64 = 0;
+
+fn dropDiffPieces() void {
+ for (&diff_pieces) |*slot| {
+ if (slot.*) |piece| syntax_allocator.free(piece.styles);
+ slot.* = null;
+ }
+ diff_clock = 0;
+}
+
+/// The styles of `content[start..end)` read as a unified diff: headers
+/// muted, `@@` lines as keywords, and each hunk's code in the language of
+/// the file its section names. A hunk's old side (context and removed
+/// lines) and new side (context and added lines) are each parsed whole, so
+/// a string or comment that spans lines colours as it does in the file.
+/// Added and removed lines carry `diff_added`/`diff_removed`. A file in no
+/// language pardes knows keeps the plain colours: added lines as strings,
+/// removed ones as numbers.
+pub fn highlightDiff(gpa: std.mem.Allocator, content: []const u8, start_raw: usize, end_raw: usize) ![]u8 {
+ return highlightDiffRows(gpa, content, start_raw, end_raw, false);
+}
+
+/// The same for a terminal's rows, where a row the terminal wrapped
+/// continues the diff line above it.
+pub fn highlightDiffRows(gpa: std.mem.Allocator, content: []const u8, start_raw: usize, end_raw: usize, continued_rows: bool) ![]u8 {
+ const tz = tracy.zone(@src(), "highlightDiff");
+ defer tz.end();
+ const start_byte = @min(start_raw, content.len);
+ const end_byte = @max(start_byte, @min(end_raw, content.len));
+ const styles = try gpa.alloc(u8, end_byte - start_byte);
errdefer gpa.free(styles);
@memset(styles, 0);
+ const Window = struct {
+ styles: []u8,
+ start: usize,
+ fn put(w: @This(), at: usize, bytes: []const u8) void {
+ const lo = @max(at, w.start);
+ const hi = @min(at + bytes.len, w.start + w.styles.len);
+ if (lo < hi) @memcpy(w.styles[lo - w.start .. hi - w.start], bytes[lo - at .. hi - at]);
+ }
+ fn fill(w: @This(), at: usize, len: usize, style: u8) void {
+ const lo = @max(at, w.start);
+ const hi = @min(at + len, w.start + w.styles.len);
+ if (lo < hi) @memset(w.styles[lo - w.start .. hi - w.start], style);
+ }
+ };
+ const window: Window = .{ .styles = styles, .start = start_byte };
+
+ var walk: diff.Walk = .{ .continued_rows = continued_rows };
+ // The run of hunk lines not painted yet, with each line's kind.
+ var piece_start: ?usize = null;
+ var piece_end: usize = 0;
+ var piece_path: []const u8 = "";
+ var kinds: [diff_piece_cut]diff.Kind = undefined;
+ var nkinds: usize = 0;
+ var at = diff.anchorBefore(content, start_byte);
+ while (true) {
+ const nl = std.mem.indexOfScalarPos(u8, content, at, '\n') orelse content.len;
+ const got = walk.step(content[at..nl]);
+ const body = switch (got.kind) {
+ .context, .added, .removed, .no_newline => true,
+ // The next hunk of the section goes on in the same piece.
+ .hunk => piece_start != null and nkinds < diff_piece_lines,
+ else => false,
+ };
+ if (piece_start) |ps| if (!body or nkinds == diff_piece_cut) {
+ if (piece_end >= start_byte) try paintDiffPiece(gpa, window, content[ps..piece_end], ps, kinds[0..nkinds], piece_path);
+ piece_start = null;
+ nkinds = 0;
+ };
+ if (at >= end_byte and piece_start == null) break;
+ if (body and (piece_start != null or got.kind != .hunk)) {
+ if (piece_start == null) {
+ piece_start = at;
+ piece_path = walk.path();
+ }
+ kinds[nkinds] = got.kind;
+ nkinds += 1;
+ piece_end = nl;
+ } else if (nl >= start_byte) {
+ const style: Syn = switch (got.kind) {
+ .hunk => .keyword,
+ .meta, .old_path, .new_path => .comment,
+ else => .none,
+ };
+ if (style != .none) window.fill(at, nl - at, @backingInt(style));
+ }
+ if (nl >= content.len) break;
+ at = nl + 1;
+ }
+ if (piece_start) |ps| if (piece_end >= start_byte)
+ try paintDiffPiece(gpa, window, content[ps..piece_end], ps, kinds[0..nkinds], piece_path);
+ return styles;
+}
+fn paintDiffPiece(gpa: std.mem.Allocator, window: anytype, bytes: []const u8, at: usize, kinds: []const diff.Kind, path: []const u8) !void {
+ const selected: ?Selected = if (enabled and syntax_started) (forExt(std.fs.path.extension(path)) catch null) else null;
+ if (selected) |sel| {
+ const hash = std.hash.Wyhash.hash(std.hash.Wyhash.hash(0, sel.name), bytes);
+ diff_clock += 1;
+ for (&diff_pieces) |*slot| if (slot.*) |*piece| if (piece.hash == hash and piece.styles.len == bytes.len) {
+ piece.used = diff_clock;
+ return window.put(at, piece.styles);
+ };
+ const colored = try syntax_allocator.alloc(u8, bytes.len);
+ errdefer syntax_allocator.free(colored);
+ try paintHunk(gpa, colored, bytes, kinds, sel);
+ window.put(at, colored);
+ // An empty slot, else the least recently used.
+ var oldest: usize = 0;
+ for (&diff_pieces, 0..) |*slot, i| {
+ const piece = slot.* orelse {
+ oldest = i;
+ break;
+ };
+ if (piece.used < diff_pieces[oldest].?.used) oldest = i;
+ }
+ if (diff_pieces[oldest]) |old| syntax_allocator.free(old.styles);
+ diff_pieces[oldest] = .{ .hash = hash, .styles = colored, .used = diff_clock };
+ return;
+ }
+ // No language: each line in one colour, as diffs were always shown.
+ var lines = std.mem.splitScalar(u8, bytes, '\n');
var offset: usize = 0;
- var lines = std.mem.splitScalar(u8, source, '\n');
- while (lines.next()) |line| {
- const syn = diffLineSyn(line);
- if (syn != .none) @memset(styles[offset .. offset + line.len], @intFromEnum(syn));
+ for (kinds) |kind| {
+ const line = lines.next() orelse break;
+ const style: u8 = switch (kind) {
+ .added => @backingInt(Syn.string) | diff_added,
+ .removed => @backingInt(Syn.number) | diff_removed,
+ .no_newline => @backingInt(Syn.comment),
+ .hunk => @backingInt(Syn.keyword),
+ else => 0,
+ };
+ if (style != 0) window.fill(at + offset, line.len, style);
offset += line.len + 1;
}
- return styles;
}
-fn diffLineSyn(line: []const u8) Syn {
- if (std.mem.startsWith(u8, line, "@@")) return .keyword;
- if (std.mem.startsWith(u8, line, "+++") or
- std.mem.startsWith(u8, line, "---") or
- std.mem.startsWith(u8, line, "diff ") or
- std.mem.startsWith(u8, line, "index ") or
- std.mem.startsWith(u8, line, "\\ No newline")) return .comment;
- if (line.len == 0) return .none;
- if (line[0] == '+') return .string;
- if (line[0] == '-') return .number;
- return .none;
+/// One piece's styles: its old side and its new side each parsed as one
+/// text, and every line's code coloured from its side, behind its
+/// one-character prefix. A `@@` line between its hunks is in neither.
+fn paintHunk(gpa: std.mem.Allocator, out: []u8, bytes: []const u8, kinds: []const diff.Kind, sel: Selected) !void {
+ @memset(out, 0);
+ const old_src = try gpa.alloc(u8, bytes.len);
+ defer gpa.free(old_src);
+ const new_src = try gpa.alloc(u8, bytes.len);
+ defer gpa.free(new_src);
+ var old_len: usize = 0;
+ var new_len: usize = 0;
+ var lines = std.mem.splitScalar(u8, bytes, '\n');
+ for (kinds) |kind| {
+ const line = lines.next() orelse break;
+ const code = line[codeStart(line, kind)..];
+ // A row a terminal wrapped goes on the line above, as it was printed.
+ const joined = continuesRow(line, kind);
+ if (kind == .context or kind == .removed) {
+ if (joined and old_len > 0) old_len -= 1;
+ @memcpy(old_src[old_len..][0..code.len], code);
+ old_len += code.len;
+ old_src[old_len] = '\n';
+ old_len += 1;
+ }
+ if (kind == .context or kind == .added) {
+ if (joined and new_len > 0) new_len -= 1;
+ @memcpy(new_src[new_len..][0..code.len], code);
+ new_len += code.len;
+ new_src[new_len] = '\n';
+ new_len += 1;
+ }
+ }
+ // A side is parsed only when a line takes its colours from it: the old
+ // side for removed lines, the new side for added lines, and context
+ // from the new side, else from the old. A piece that only adds or only
+ // removes is one parse.
+ var removes = false;
+ var adds = false;
+ for (kinds) |kind| {
+ removes = removes or kind == .removed;
+ adds = adds or kind == .added;
+ }
+ const context_new = adds or !removes;
+ const old_colors = try gpa.alloc(u8, old_len);
+ defer gpa.free(old_colors);
+ @memset(old_colors, 0);
+ if (!context_new or removes) paint(old_colors, old_src[0..old_len], sel);
+ const new_colors = try gpa.alloc(u8, new_len);
+ defer gpa.free(new_colors);
+ @memset(new_colors, 0);
+ if (context_new) paint(new_colors, new_src[0..new_len], sel);
+
+ lines = std.mem.splitScalar(u8, bytes, '\n');
+ var offset: usize = 0;
+ var old_at: usize = 0;
+ var new_at: usize = 0;
+ for (kinds) |kind| {
+ const line = lines.next() orelse break;
+ defer offset += line.len + 1;
+ const skip = codeStart(line, kind);
+ const code_len = line.len - skip;
+ const dst = out[offset + skip ..][0..code_len];
+ if (continuesRow(line, kind)) {
+ if ((kind == .context or kind == .removed) and old_at > 0) old_at -= 1;
+ if ((kind == .context or kind == .added) and new_at > 0) new_at -= 1;
+ }
+ switch (kind) {
+ .removed => @memcpy(dst, old_colors[old_at..][0..code_len]),
+ .added => @memcpy(dst, new_colors[new_at..][0..code_len]),
+ .context => @memcpy(dst, if (context_new) new_colors[new_at..][0..code_len] else old_colors[old_at..][0..code_len]),
+ .no_newline => @memset(out[offset..][0..line.len], @backingInt(Syn.comment)),
+ .hunk => @memset(out[offset..][0..line.len], @backingInt(Syn.keyword)),
+ else => {},
+ }
+ if (kind == .context or kind == .removed) old_at += code_len + 1;
+ if (kind == .context or kind == .added) new_at += code_len + 1;
+ const flag: u8 = switch (kind) {
+ .added => diff_added,
+ .removed => diff_removed,
+ else => 0,
+ };
+ if (flag != 0) for (out[offset..][0..line.len]) |*b| {
+ b.* |= flag;
+ };
+ }
+}
+
+fn continuesRow(line: []const u8, kind: diff.Kind) bool {
+ return line.len > 0 and codeStart(line, kind) == 0 and
+ (kind == .context or kind == .added or kind == .removed);
+}
+
+/// Where a hunk line's code starts: past its prefix, which a blank context
+/// line (its space stripped) and a terminal's wrapped row have none of.
+fn codeStart(line: []const u8, kind: diff.Kind) usize {
+ if (line.len == 0) return 0;
+ return switch (kind) {
+ .context => @intFromBool(line[0] == ' '),
+ .added => @intFromBool(line[0] == '+'),
+ .removed => @intFromBool(line[0] == '-'),
+ else => 0,
+ };
}
test "syntax a results row is coloured by the file its location names" {
@@ -904,8 +1412,8 @@ test "syntax a results row is coloured by the file its location names" {
try std.testing.expectEqual(content.len, styles.len);
for (styles[0.."src/a.zig:1:1".len]) |b| try std.testing.expectEqual(@as(u8, 0), b);
const zig_kw = std.mem.indexOf(u8, content, "struct").?;
- try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[zig_kw]);
- try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[zig_kw + 5]);
+ try std.testing.expectEqual(@backingInt(Syn.keyword), styles[zig_kw]);
+ try std.testing.expectEqual(@backingInt(Syn.keyword), styles[zig_kw + 5]);
const prose = std.mem.indexOf(u8, content, "just some prose").?;
for (styles[prose .. prose + 14]) |b| try std.testing.expectEqual(@as(u8, 0), b);
const bare = std.mem.indexOf(u8, content, "src/c.zig").?;
@@ -933,25 +1441,11 @@ test "syntax codeAfterLocation takes whole-token locations and nothing else" {
try std.testing.expect(codeAfterLocation("src/x.zig:7:2") == null);
try std.testing.expect(codeAfterLocation("") == null);
try std.testing.expect(codeAfterLocation(" leading space") == null);
-}
-
-test "syntax tree-sitter allocator callbacks preserve and free exact allocations" {
- syntax_allocator = std.testing.allocator;
- defer syntax_allocator = undefined;
-
- var live: ?*anyopaque = syntaxCalloc(4, 1) orelse return error.OutOfMemory;
- defer if (live) |pointer| syntaxFree(pointer);
- const original: [*]u8 = @ptrCast(live.?);
- try std.testing.expectEqualSlices(u8, &.{ 0, 0, 0, 0 }, original[0..4]);
- @memcpy(original[0..4], "data");
-
- live = syntaxRealloc(live, 32) orelse return error.OutOfMemory;
- const grown: [*]u8 = @ptrCast(live.?);
- try std.testing.expectEqualSlices(u8, "data", grown[0..4]);
-
- try std.testing.expect(syntaxCalloc(std.math.maxInt(usize), 2) == null);
- try std.testing.expect(syntaxRealloc(live, 0) == null);
- live = null;
+ const indented = " some directory/x.zig:7:2-9 return 42;";
+ const nested = codeAfterLocation(indented) orelse return error.ShouldBeALocation;
+ try std.testing.expectEqualStrings("some directory/x.zig", nested.path);
+ try std.testing.expectEqualStrings(" return 42;", nested.text);
+ try std.testing.expectEqualStrings(nested.text, indented[nested.at..]);
}
test "syntax default full grammar set highlights Typst source" {
@@ -967,14 +1461,14 @@ test "syntax default full grammar set highlights Typst source" {
const keyword_at = std.mem.indexOf(u8, source, "let").?;
const number_at = std.mem.indexOf(u8, source, "42").?;
const string_at = std.mem.indexOf(u8, source, "\"hello\"").?;
- try std.testing.expectEqual(Syn.comment, @as(Syn, @enumFromInt(styles[comment_at])));
- try std.testing.expectEqual(Syn.keyword, @as(Syn, @enumFromInt(styles[keyword_at])));
- try std.testing.expectEqual(Syn.number, @as(Syn, @enumFromInt(styles[number_at])));
- try std.testing.expectEqual(Syn.string, @as(Syn, @enumFromInt(styles[string_at])));
+ try std.testing.expectEqual(Syn.comment, @as(Syn, @fromBackingInt(@intCast(styles[comment_at]))));
+ try std.testing.expectEqual(Syn.keyword, @as(Syn, @fromBackingInt(@intCast(styles[keyword_at]))));
+ try std.testing.expectEqual(Syn.number, @as(Syn, @fromBackingInt(@intCast(styles[number_at]))));
+ try std.testing.expectEqual(Syn.string, @as(Syn, @fromBackingInt(@intCast(styles[string_at]))));
const short_ext = try highlightFileRange(std.testing.allocator, "paper.typ", source, 0, source.len);
defer std.testing.allocator.free(short_ext);
- try std.testing.expectEqual(Syn.keyword, @as(Syn, @enumFromInt(short_ext[keyword_at])));
+ try std.testing.expectEqual(Syn.keyword, @as(Syn, @fromBackingInt(@intCast(short_ext[keyword_at]))));
}
test "syntax Typst markup constructs paint and raw blocks inject their language" {
@@ -999,7 +1493,7 @@ test "syntax Typst markup constructs paint and raw blocks inject their language"
const synAt = struct {
fn f(s: []const u8, src: []const u8, needle: []const u8, offset: usize) Syn {
- return @enumFromInt(s[std.mem.indexOf(u8, src, needle).? + offset]);
+ return @fromBackingInt(@intCast(s[std.mem.indexOf(u8, src, needle).? + offset]));
}
}.f;
try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "= Heading", 0));
@@ -1039,7 +1533,7 @@ test "syntax markdown highlights markup, injects inline spans and fenced code bl
const synAt = struct {
fn f(s: []const u8, src: []const u8, needle: []const u8, offset: usize) Syn {
- return @enumFromInt(s[std.mem.indexOf(u8, src, needle).? + offset]);
+ return @fromBackingInt(@intCast(s[std.mem.indexOf(u8, src, needle).? + offset]));
}
}.f;
try std.testing.expectEqual(Syn.keyword, synAt(styles, source, "Title", 0));
@@ -1067,8 +1561,8 @@ test "syntax inline fast path agrees with full Markdown query" {
const actual = try std.testing.allocator.alloc(u8, source.len);
defer std.testing.allocator.free(actual);
for ([_]Syn{ .none, .keyword }) |background| {
- @memset(expected, @intFromEnum(background));
- @memset(actual, @intFromEnum(background));
+ @memset(expected, @backingInt(background));
+ @memset(actual, @backingInt(background));
runQuery(expected, sel, tree, 0);
inject(actual, source, tree.rootNode(), true);
if (!std.mem.eql(u8, expected, actual)) std.debug.print("inline mismatch: {s}\n", .{source});
@@ -1107,40 +1601,224 @@ test "syntax plain Markdown keeps block styles without starting the inline parse
const source = "# Heading\n\nPlain prose.\n\n indented code\n";
const styles = try highlightFileRange(std.testing.allocator, "a.md", source, 0, source.len);
defer std.testing.allocator.free(styles);
- try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[2]);
- try std.testing.expectEqual(@intFromEnum(Syn.none), styles[std.mem.indexOf(u8, source, "Plain").?]);
- try std.testing.expectEqual(@intFromEnum(Syn.string), styles[std.mem.indexOf(u8, source, "indented").?]);
+ try std.testing.expectEqual(@backingInt(Syn.keyword), styles[2]);
+ try std.testing.expectEqual(@backingInt(Syn.none), styles[std.mem.indexOf(u8, source, "Plain").?]);
+ try std.testing.expectEqual(@backingInt(Syn.string), styles[std.mem.indexOf(u8, source, "indented").?]);
for (specs) |spec| {
if (std.mem.eql(u8, spec.name, "markdown_inline")) try std.testing.expect(spec.selected == null);
}
}
test "syntax highlightDiff colors unified diff lines by prefix" {
- const diff =
+ const diff_text =
"diff --git a/x b/x\n" ++
"--- a/x\n" ++
"+++ b/x\n" ++
- "@@ -1,3 +1,3 @@\n" ++
+ "@@ -1,2 +1,2 @@\n" ++
" context\n" ++
"-old line\n" ++
"+new line\n";
- const styles = try highlightDiff(std.testing.allocator, diff, 0, diff.len);
+ const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len);
defer std.testing.allocator.free(styles);
+ try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "diff --git", 0));
+ try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "--- a/x", 0));
+ try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "+++ b/x", 0));
+ try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "@@ -1,2", 0));
+ try std.testing.expectEqual(Syn.none, diffSynAt(styles, diff_text, " context", 0));
+ // A file in no known language: each line in one colour, and tinted.
+ try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "-old line", 0));
+ try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "+new line", 0));
+ try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "-old line").?] & diff_removed != 0);
+ try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "+new line").? + 4] & diff_added != 0);
+ try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, " context").?] & (diff_added | diff_removed) == 0);
+}
- const byteSyn = struct {
- fn at(s: []const u8, src: []const u8, needle: []const u8) Syn {
- const i = std.mem.indexOf(u8, src, needle).?;
- return @enumFromInt(s[i]);
- }
- }.at;
+fn diffSynAt(styles: []const u8, text: []const u8, needle: []const u8, offset: usize) Syn {
+ return @fromBackingInt(@intCast(styles[std.mem.indexOf(u8, text, needle).? + offset] & syn_bits));
+}
- try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "diff --git"));
- try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "--- a/x"));
- try std.testing.expectEqual(Syn.comment, byteSyn(styles, diff, "+++ b/x"));
- try std.testing.expectEqual(Syn.keyword, byteSyn(styles, diff, "@@ -1,3"));
- try std.testing.expectEqual(Syn.none, byteSyn(styles, diff, " context"));
- try std.testing.expectEqual(Syn.number, byteSyn(styles, diff, "-old line"));
- try std.testing.expectEqual(Syn.string, byteSyn(styles, diff, "+new line"));
+test "syntax a Zig hunk is coloured as Zig, each side parsed whole" {
+ if (!enabled) return;
+ start(std.testing.allocator);
+ defer stop();
+ const diff_text =
+ "diff --git a/src/shape.zig b/src/shape.zig\n" ++
+ "index 1111111..2222222 100644\n" ++
+ "--- a/src/shape.zig\n" ++
+ "+++ b/src/shape.zig\n" ++
+ "@@ -1,6 +1,6 @@\n" ++
+ " const Shape = struct {\n" ++
+ "- side: u8 = 7,\n" ++
+ "+ side: u16 = 42,\n" ++
+ " const note =\n" ++
+ "- \\\\old words defer\n" ++
+ "+ \\\\new words defer\n" ++
+ " ;\n" ++
+ " };\n";
+ const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len);
+ defer std.testing.allocator.free(styles);
+ try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "struct", 0));
+ try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "7,", 0));
+ try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "42", 0));
+ // A multiline string, told apart only by parsing the side it is on:
+ // `defer` inside it is no keyword.
+ try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "old words defer", 10));
+ try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "new words defer", 10));
+ // The prefix is no code: it carries the tint alone.
+ const added = std.mem.indexOf(u8, diff_text, "+ side").?;
+ try std.testing.expectEqual(diff_added, styles[added]);
+ try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "42").?] & diff_added != 0);
+ try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "7,").?] & diff_removed != 0);
+ try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "index 111", 0));
+
+ // A window in the middle is coloured as the whole is.
+ const from = std.mem.indexOf(u8, diff_text, " const note").?;
+ const part = try highlightDiff(std.testing.allocator, diff_text, from, diff_text.len);
+ defer std.testing.allocator.free(part);
+ try std.testing.expectEqualSlices(u8, styles[from..], part);
+}
+
+test "syntax a Python hunk is coloured as Python" {
+ if (!enabled or !full_grammars) return;
+ start(std.testing.allocator);
+ defer stop();
+ const diff_text =
+ "--- a/tool.py\n" ++
+ "+++ b/tool.py\n" ++
+ "@@ -1,3 +1,4 @@\n" ++
+ " def run(n):\n" ++
+ "- return n\n" ++
+ "+ # doubled now\n" ++
+ "+ return n * 2\n" ++
+ " \n";
+ const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len);
+ defer std.testing.allocator.free(styles);
+ try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "def", 0));
+ try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "return n\n", 0));
+ try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "# doubled", 2));
+ try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "2\n", 0));
+}
+
+test "syntax a deleted file's lines are coloured in its language" {
+ if (!enabled) return;
+ start(std.testing.allocator);
+ defer stop();
+ const diff_text =
+ "diff --git a/gone.zig b/gone.zig\n" ++
+ "deleted file mode 100644\n" ++
+ "--- a/gone.zig\n" ++
+ "+++ /dev/null\n" ++
+ "@@ -1,2 +0,0 @@\n" ++
+ "-const Gone = enum { a };\n" ++
+ "-// the end\n";
+ const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len);
+ defer std.testing.allocator.free(styles);
+ try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "enum", 0));
+ try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "// the end", 3));
+ try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "enum").?] & diff_removed != 0);
+ try std.testing.expectEqual(Syn.comment, diffSynAt(styles, diff_text, "deleted file", 0));
+}
+
+test "syntax a hunk in no known language keeps the line colours" {
+ if (!enabled) return;
+ start(std.testing.allocator);
+ defer stop();
+ const diff_text =
+ "--- a/notes.unknownext\n" ++
+ "+++ b/notes.unknownext\n" ++
+ "@@ -1 +1 @@\n" ++
+ "-const struct 1\n" ++
+ "+const struct 2\n";
+ const styles = try highlightDiff(std.testing.allocator, diff_text, 0, diff_text.len);
+ defer std.testing.allocator.free(styles);
+ try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "struct 1", 0));
+ try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "struct 2", 0));
+ try std.testing.expectEqual(Syn.string, diffSynAt(styles, diff_text, "+const struct 2", 0));
+}
+
+test "syntax git diff output in a command pane is told and coloured by its rows" {
+ if (!enabled) return;
+ start(std.testing.allocator);
+ defer stop();
+ // A command pane's rows: its command line, then git's output, with one
+ // long added line the terminal wrapped onto a second row.
+ const rows = [_][]const u8{
+ "git diff",
+ "diff --git a/main.zig b/main.zig",
+ "--- a/main.zig",
+ "+++ b/main.zig",
+ "@@ -1,2 +1,2 @@",
+ " pub fn main() void {",
+ "+ const answer: u32 = 42; // the ",
+ "wrapped rest",
+ "- return;",
+ "",
+ };
+ try std.testing.expect(diff.looksLikeDiff(&rows));
+ try std.testing.expect(!diff.looksLikeDiff(rows[5..]));
+ const text = try std.mem.join(std.testing.allocator, "\n", &rows);
+ defer std.testing.allocator.free(text);
+ const styles = try highlightDiffRows(std.testing.allocator, text, 0, text.len, true);
+ defer std.testing.allocator.free(styles);
+ try std.testing.expectEqual(Syn.none, diffSynAt(styles, text, "git diff", 0));
+ try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, text, "fn main", 0));
+ try std.testing.expectEqual(Syn.number, diffSynAt(styles, text, "42", 0));
+ // The wrapped row stays an added line, and inside the comment.
+ try std.testing.expect(styles[std.mem.indexOf(u8, text, "wrapped rest").?] & diff_added != 0);
+ try std.testing.expectEqual(Syn.comment, diffSynAt(styles, text, "wrapped rest", 0));
+ try std.testing.expect(styles[std.mem.indexOf(u8, text, "return").?] & diff_removed != 0);
+}
+
+test "syntax a section's small hunks share one parse" {
+ if (!enabled) return;
+ const gpa = std.testing.allocator;
+ start(gpa);
+ defer stop();
+ const diff_text =
+ "--- a/x.zig\n+++ b/x.zig\n" ++
+ "@@ -1 +1 @@\n-const a = 1;\n+const a = 2;\n" ++
+ "@@ -10 +10 @@\n-const b = 1;\n+const b = 2;\n" ++
+ "@@ -20 +20 @@\n-const c = 1;\n+const c = 2;\n" ++
+ "--- a/y.zig\n+++ b/y.zig\n" ++
+ "@@ -1 +1 @@\n-const d = 1;\n+const d = 2;\n";
+ const styles = try highlightDiff(gpa, diff_text, 0, diff_text.len);
+ defer gpa.free(styles);
+ var cached: usize = 0;
+ for (diff_pieces) |slot| cached += @intFromBool(slot != null);
+ try std.testing.expectEqual(@as(usize, 2), cached); // x.zig's three hunks, and y.zig's
+ try std.testing.expectEqual(Syn.keyword, diffSynAt(styles, diff_text, "@@ -10", 0));
+ try std.testing.expectEqual(Syn.number, diffSynAt(styles, diff_text, "2;\n@@ -20", 0));
+ try std.testing.expect(styles[std.mem.indexOf(u8, diff_text, "@@ -10").?] & (diff_added | diff_removed) == 0);
+}
+
+test "syntax a hunk painted once is kept, and a long one is parsed in pieces" {
+ if (!enabled) return;
+ const gpa = std.testing.allocator;
+ start(gpa);
+ defer stop();
+ var text: std.Io.Writer.Allocating = .init(gpa);
+ defer text.deinit();
+ const n = diff_piece_cut * 20 + 10;
+ try text.writer.print("--- /dev/null\n+++ b/big.zig\n@@ -0,0 +1,{d} @@\n", .{n});
+ for (0..n) |i| try text.writer.print("+const v{d} = {d};\n", .{ i, i });
+ const content = text.written();
+ const at = std.mem.indexOf(u8, content, "+const v1000 ").?;
+ const window = try highlightDiff(gpa, content, at, at + 200);
+ defer gpa.free(window);
+ try std.testing.expectEqual(Syn.number, @as(Syn, @fromBackingInt(@intCast(window["+const v1000 = ".len] & syn_bits))));
+ var cached: usize = 0;
+ for (diff_pieces) |slot| if (slot) |piece| {
+ cached += 1;
+ // One piece of many, not the whole hunk.
+ try std.testing.expect(piece.styles.len < content.len / 4);
+ };
+ try std.testing.expectEqual(@as(usize, 1), cached);
+ const again = try highlightDiff(gpa, content, at, at + 200);
+ defer gpa.free(again);
+ try std.testing.expectEqualSlices(u8, window, again);
+ cached = 0;
+ for (diff_pieces) |slot| cached += @intFromBool(slot != null);
+ try std.testing.expectEqual(@as(usize, 1), cached);
}
test "syntax result fragments preserve source indentation and inline markup" {
@@ -1149,6 +1827,7 @@ test "syntax result fragments preserve source indentation and inline markup" {
defer stop();
const fixtures = [_]struct { path: []const u8, source: []const u8 }{
.{ .path = "a.zig", .source = " const number = 42; // note" },
+ .{ .path = " nested path/a.zig", .source = " return call(42, \"hello\"); // note" },
.{ .path = "a.md", .source = "# Heading *slant*" },
.{ .path = "a.md", .source = "**bold** and `code`" },
.{ .path = "a.md", .source = " # this is indented code" },
@@ -1191,7 +1870,7 @@ test "syntax query filtering preserves upstream colors" {
reference.query = raw_query;
reference.capture_styles = try std.testing.allocator.alloc(u8, raw_query.captureCount());
defer std.testing.allocator.free(reference.capture_styles);
- for (reference.capture_styles, 0..) |*style, id| style.* = @intFromEnum(synFor(raw_query.captureNameForId(@intCast(id)) orelse ""));
+ for (reference.capture_styles, 0..) |*style, id| style.* = @backingInt(synFor(raw_query.captureNameForId(@intCast(id)) orelse ""));
const tree = selected.parser.parseString(source, null) orelse return error.ParseFailed;
defer tree.destroy();
var expected: [source.len]u8 = @splat(0);
@@ -1217,8 +1896,8 @@ test "syntax Zig keyword captures cover both bytes beyond line ten thousand" {
defer std.testing.allocator.free(styles);
for ([_][]const u8{ "fn", "if" }) |keyword| {
const at = std.mem.indexOf(u8, code, keyword).?;
- try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[at]);
- try std.testing.expectEqual(@intFromEnum(Syn.keyword), styles[at + 1]);
+ try std.testing.expectEqual(@backingInt(Syn.keyword), styles[at]);
+ try std.testing.expectEqual(@backingInt(Syn.keyword), styles[at + 1]);
}
}
@@ -1234,8 +1913,8 @@ test "syntax allocator switching clears default-runtime caches" {
const restored = try highlightFileRange(std.testing.allocator, "a.zig", source, 0, source.len);
defer std.testing.allocator.free(restored);
defer stop();
- try std.testing.expectEqual(@intFromEnum(Syn.keyword), restored[0]);
- try std.testing.expectEqual(@intFromEnum(Syn.keyword), restored[1]);
+ try std.testing.expectEqual(@backingInt(Syn.keyword), restored[0]);
+ try std.testing.expectEqual(@backingInt(Syn.keyword), restored[1]);
}
test "syntax location context snapshots retain omitted multiline comment scope" {
@@ -1259,8 +1938,8 @@ test "syntax location context snapshots retain omitted multiline comment scope"
const styles = try highlightLocationRows(gpa, content, &rows);
defer gpa.free(styles);
for (styles[rows[0].code_start..][0..code.len]) |style|
- try std.testing.expectEqual(@intFromEnum(Syn.comment), style);
- try std.testing.expectEqual(@intFromEnum(Syn.none), styles[0]);
+ try std.testing.expectEqual(@backingInt(Syn.comment), style);
+ try std.testing.expectEqual(@backingInt(Syn.none), styles[0]);
}
test "syntax declaration locations clear snapshots without muting ordinary context" {
@@ -1270,7 +1949,7 @@ test "syntax declaration locations clear snapshots without muting ordinary conte
defer stop();
const header = "pub fn run() void {";
const content = "| \t" ++ header ++ "\n| \t const value = 42;\nfile.zig:3\t}\n";
- const snapshots = [_]u8{0xff} ** header.len;
+ const snapshots = @as([header.len]u8, @splat(0xff));
const Row = struct {
path: []const u8 = "file.zig",
code_start: usize,
@@ -1286,7 +1965,7 @@ test "syntax declaration locations clear snapshots without muting ordinary conte
const styles = try highlightLocationRows(gpa, content, &rows);
defer gpa.free(styles);
for (styles[3..][0..header.len]) |style| try std.testing.expectEqual(@as(u8, 0), style);
- try std.testing.expectEqual(@intFromEnum(Syn.number), styles[std.mem.indexOf(u8, content, "42").?]);
+ try std.testing.expectEqual(@backingInt(Syn.number), styles[std.mem.indexOf(u8, content, "42").?]);
}
test "syntax stacked addresses do not split multiline source groups" {
@@ -1306,11 +1985,11 @@ test "syntax stacked addresses do not split multiline source groups" {
};
const styles = try highlightLocationRows(gpa, content, &rows);
defer gpa.free(styles);
- try std.testing.expectEqual(@intFromEnum(Syn.comment), styles[std.mem.indexOf(u8, content, "42").?]);
- try std.testing.expectEqual(@intFromEnum(Syn.number), styles[std.mem.indexOf(u8, content, "7;").?]);
+ try std.testing.expectEqual(@backingInt(Syn.comment), styles[std.mem.indexOf(u8, content, "42").?]);
+ try std.testing.expectEqual(@backingInt(Syn.number), styles[std.mem.indexOf(u8, content, "7;").?]);
for ([_][]const u8{ "a.c:1:1", "a.c:2:1", "a.c:3:1" }) |label| {
const at = std.mem.indexOf(u8, content, label).?;
- for (styles[at..][0..label.len]) |style| try std.testing.expectEqual(@intFromEnum(Syn.none), style);
+ for (styles[at..][0..label.len]) |style| try std.testing.expectEqual(@backingInt(Syn.none), style);
}
}
@@ -1374,3 +2053,72 @@ test "syntax source analysis preserves declarations and injected colors" {
try std.testing.expectEqual(@as(usize, 0), unsupported.declarations.len);
try std.testing.expectEqual(@as(usize, 0), unsupported.colors.len);
}
+
+test "tables are the pipe_tables tree-sitter finds" {
+ if (!enabled or !full_grammars) return;
+ start(std.testing.allocator);
+ defer stop();
+ const Tables = @import("Tables.zig");
+ const source =
+ "# Tables\n" ++
+ "\n" ++
+ "| a | b |\n" ++
+ "|---|:-:|\n" ++
+ "| 1 | 2 |\n" ++
+ "no pipe row\n" ++
+ "\n" ++
+ "a | b\n" ++
+ "--|--\n" ++
+ "c | d\n" ++
+ "# heading ends it\n" ++
+ "| x |\n" ++
+ "|---|\n" ++
+ "\n" ++
+ "```\n" ++
+ "| in | fence |\n" ++
+ "|----|-------|\n" ++
+ "```\n" ++
+ "\n" ++
+ "| one | two |\n" ++
+ "|-----|\n" ++
+ "\n" ++
+ " | indented | table |\n" ++
+ " | -------- | ----- |\n" ++
+ " | row | here |\n" ++
+ "> quote ends\n" ++
+ "\n" ++
+ "| esc \\| aped | two |\n" ++
+ "| --- | --- |\n" ++
+ "- list ends\n";
+ const selected = (try forLang("markdown")).?;
+ const tree = selected.parser.parseString(source, null) orelse return error.ParseFailed;
+ defer tree.destroy();
+ // Every pipe_table node's lines, wherever it is in the tree.
+ var found: std.ArrayList([2]u32) = .empty;
+ defer found.deinit(std.testing.allocator);
+ var stack: std.ArrayList(ts.Node) = .empty;
+ defer stack.deinit(std.testing.allocator);
+ try stack.append(std.testing.allocator, tree.rootNode());
+ while (stack.pop()) |node| {
+ if (std.mem.eql(u8, node.kind(), "pipe_table")) {
+ const end = node.endPoint();
+ try found.append(std.testing.allocator, .{ node.startPoint().row, end.row + @intFromBool(end.column > 0) });
+ continue;
+ }
+ var i = node.childCount();
+ while (i > 0) : (i -= 1) if (node.child(i - 1)) |child| try stack.append(std.testing.allocator, child);
+ }
+ // The tables, the indented one and the escaped pipe's; not the fence's
+ // or the one whose delimiter has a cell too few.
+ try std.testing.expectEqual(@as(usize, 5), found.items.len);
+ var t: Tables = .{};
+ defer t.deinit(std.testing.allocator);
+ try t.scan(std.testing.allocator, source);
+ var mine: std.ArrayList([2]u32) = .empty;
+ defer mine.deinit(std.testing.allocator);
+ for (t.list.items) |table| try mine.append(std.testing.allocator, .{ table.first, table.end });
+ std.testing.expectEqualSlices([2]u32, found.items, mine.items) catch |e| {
+ std.debug.print("tree-sitter {any}\nTables {any}\n", .{ found.items, mine.items });
+ return e;
+ };
+}