//! A program's output, paged into a +Pager (`pardes -`): the text it
//! prints, and its SGR colours as spans over that text. It reads the bytes
//! as a terminal does (ECMA-48's parse as ghostty-vt's table has it, UTF-8
//! with each ill-formed part one U+FFFD) and keeps what is printed, how SGR
//! styles it and the controls that move along a line (CR, LF, BS, HT and
//! CSI's CHT, REP and the return in CNL and CPL); every other sequence is
//! read to its end and dropped (cursor motion, OSC titles and hyperlinks,
//! DCS, APC, modes). The spans are display state of the pane only: no 9P
//! file reads them (File.recolorSyntax draws them).
const std = @import("std");
const pardes = @import("pardes.zig");
const CellStyle = @import("surface.zig").CellStyle;
const Color = @import("surface.zig").Color;
/// Bytes `start..end` of the text, drawn in `style`. A palette colour stays
/// `.index`, resolved through the theme as it is drawn (terminal.palColor),
/// and `.default` is the pane's own.
pub const Span = struct { start: u32, end: u32, style: CellStyle };
/// ponytail: past this many spans the rest of a page is plain, rather than
/// ~100 MB of spans for a 256 MiB coloured log; a packed style if it matters.
pub const max_spans = 4 << 20;
pub const Page = struct {
text: []u8,
spans: []Span,
pub fn deinit(page: Page, gpa: std.mem.Allocator) void {
gpa.free(page.text);
gpa.free(page.spans);
}
};
/// `raw`, a program's output for a terminal, as text and its spans. Both
/// are `gpa`'s.
pub fn parse(gpa: std.mem.Allocator, raw: []const u8) !Page {
var h: Handler = .{ .gpa = gpa };
errdefer h.text.deinit(gpa);
errdefer h.spans.deinit(gpa);
try h.text.ensureTotalCapacity(gpa, raw.len);
// The printed code points since the last control, printed as one run.
var run: [4096]u32 = undefined;
var n: usize = 0;
var state: State = .ground;
var i: usize = 0;
feed: while (i < raw.len) {
const c = raw[i];
if (state != .ground or c == 0x1b) {
h.printRun(run[0..n]);
n = 0;
state = h.step(state, c);
i += 1;
continue;
}
// Printable ASCII, most of a page, straight into the run.
if (c >= 0x20 and c < 0x7f) {
const end = @min(raw.len, i + run.len - n);
while (i < end and raw[i] >= 0x20 and raw[i] < 0x7f) : (i += 1) {
run[n] = raw[i];
n += 1;
}
if (n < run.len) continue;
h.printRun(&run);
n = 0;
continue;
}
// UTF-8: an ill-formed part (a stray byte, or the longest start of
// a sequence that does not go on) is one U+FFFD; one the input ends
// inside may yet go on, and is not printed.
var cp: u21 = c;
var len: usize = 1;
if (c >= 0x80) {
cp = 0xfffd;
if (c >= 0xc2 and c <= 0xf4) {
const want: usize = if (c < 0xe0) 2 else if (c < 0xf0) 3 else 4;
const lo: u8 = if (c == 0xe0) 0xa0 else if (c == 0xf0) 0x90 else 0x80;
const hi: u8 = if (c == 0xed) 0x9f else if (c == 0xf4) 0x8f else 0xbf;
var acc: u21 = c & (@as(u8, 0x7f) >> @intCast(want));
while (len < want) : (len += 1) {
if (i + len == raw.len) break :feed;
const b = raw[i + len];
if (b < (if (len == 1) lo else 0x80) or b > (if (len == 1) hi else 0xbf)) break;
acc = acc << 6 | (b & 0x3f);
}
if (len == want) cp = acc;
}
}
i += len;
// A C1 control that came as UTF-8 is dropped, as xterm drops it.
if (cp >= 0x20 and (cp < 0x80 or cp > 0x9f)) {
run[n] = cp;
n += 1;
if (n < run.len) continue;
}
h.printRun(run[0..n]);
n = 0;
if (cp < 0x20) h.execute(@intCast(cp));
}
h.printRun(run[0..n]);
if (h.failed) return error.OutOfMemory;
const text = try h.text.toOwnedSlice(gpa);
errdefer gpa.free(text);
return .{ .text = text, .spans = try h.spans.toOwnedSlice(gpa) };
}
/// Where the parse is in a sequence: ECMA-48's states, the DCS's data and
/// its ignored tail one (`dcs_body`) and SOS, PM and APC one (`apc`).
const State = enum { ground, esc, esc_inter, csi_entry, csi_param, csi_inter, csi_ignore, osc, dcs_entry, dcs_param, dcs_inter, dcs_body, apc };
const Handler = struct {
gpa: std.mem.Allocator,
text: std.ArrayList(u8) = .empty,
spans: std.ArrayList(Span) = .empty,
/// What SGR has set so far.
style: CellStyle = .{},
/// Where the line being printed starts, and its last character.
line_start: usize = 0,
last: usize = 0,
/// A carriage return not yet followed by a newline: the line starts
/// over at the next print, as on a terminal, so a progress line keeps
/// its last state.
cr: bool = false,
/// A backspace: the next print strikes over the character before it.
bs: bool = false,
failed: bool = false,
/// The control sequence being read: its parameters as ghostty-vt keeps
/// them (24 at most, each saturating at 65535, a bit in `colons` for
/// each one a colon ends), and whether it has an intermediate or a
/// private marker, which no sequence kept here takes.
csi: struct {
params: [24]u16 = undefined,
n: u8 = 0,
acc: u16 = 0,
digits: bool = false,
colons: u32 = 0,
inter: bool = false,
} = .{},
/// The state after byte `c` in `state` (not ground's printing, which
/// `parse` decodes), acting on the controls and sequences a page keeps.
fn step(h: *Handler, state: State, c: u8) State {
switch (c) {
0x18, 0x1a => return .ground,
0x1b => {
h.csi = .{};
return .esc;
},
// A C1 control byte, in all but an OSC's or a DCS's data.
0x80...0x9f => return if (state == .osc or state == .dcs_body) state else switch (c) {
0x90 => .dcs_entry,
0x9b => csi: {
h.csi = .{};
break :csi .csi_entry;
},
0x9d => .osc,
0x98, 0x9e, 0x9f => .apc,
else => .ground,
},
0xa0...0xff => return state,
else => {},
}
switch (state) {
.osc => return if (c == 0x07) .ground else .osc,
.apc, .dcs_body => return state,
.dcs_entry, .dcs_param, .dcs_inter => return switch (c) {
0...0x1f, 0x7f => state,
0x20...0x2f => .dcs_inter,
'0'...'9', ';' => if (state == .dcs_inter) .dcs_body else .dcs_param,
'<'...'?' => if (state == .dcs_entry) .dcs_param else .dcs_body,
else => .dcs_body,
},
else => {},
}
if (c < 0x20) {
h.execute(c);
return state;
}
if (c == 0x7f) return state;
switch (state) {
.esc => return switch (c) {
'[' => .csi_entry,
']' => .osc,
'P' => .dcs_entry,
'X', '^', '_' => .apc,
0x20...0x2f => .esc_inter,
else => .ground,
},
.esc_inter => return if (c < 0x30) state else .ground,
.csi_ignore => return if (c < 0x40) state else .ground,
else => {},
}
const p = &h.csi;
switch (c) {
0x20...0x2f => {
p.inter = true;
return .csi_inter;
},
'0'...'9', ':', ';' => {
if (state == .csi_inter or (c == ':' and state == .csi_entry)) return .csi_ignore;
if (p.n == p.params.len) return .csi_param;
if (c <= '9') {
p.acc = p.acc *| 10 +| (c - '0');
p.digits = true;
return .csi_param;
}
p.params[p.n] = p.acc;
if (c == ':') p.colons |= @as(u32, 1) << @intCast(p.n);
p.n += 1;
p.acc = 0;
p.digits = false;
return .csi_param;
},
'<'...'?' => {
if (state != .csi_entry) return .csi_ignore;
p.inter = true;
return .csi_param;
},
else => {
h.dispatch(c);
return .ground;
},
}
}
/// A C0 control: the line feeds (LF, VT and FF), CR, BS and HT.
fn execute(h: *Handler, c: u8) void {
switch (c) {
'\n', 0x0b, 0x0c => {
h.append("\n");
h.line_start = h.text.items.len;
h.last = h.line_start;
h.cr = false;
h.bs = false;
},
'\r' => h.cr = true,
0x08 => h.bs = h.text.items.len > h.line_start,
'\t' => h.print('\t'),
else => {},
}
}
/// The control sequence ending in `f`: SGR, and of the rest what moves
/// along a line, dropped as ghostty-vt drops a malformed one.
fn dispatch(h: *Handler, f: u8) void {
const p = &h.csi;
if (p.n == p.params.len) return;
if (p.digits) {
p.params[p.n] = p.acc;
p.n += 1;
}
if (p.inter or (f != 'm' and p.colons != 0)) return;
if (f == 'm') return h.sgr(p.params[0..p.n], p.colons);
if (p.n > 1) return;
const count: u16 = if (p.n == 0) 1 else p.params[0];
switch (f) {
// CNL and CPL: down or up, which a page has not, and a return.
'E', 'F' => h.cr = true,
// CHT; and REP, the last character again, as a terminal draws
// it: a terminal stops at its edge, and this at a line's worth.
'I' => for (0..@min(count, 64)) |_| h.print('\t'),
'b' => if (h.text.items.len > h.last) {
const cp = std.unicode.utf8Decode(h.text.items[h.last..]) catch return;
for (0..@min(count, 1024)) |_| h.print(cp);
},
else => {},
}
}
/// SGR's parameters into the style, read as ghostty-vt's sgr.Parser
/// reads them: a colour short of its values leaves them to be read as
/// SGRs of their own, and a colon list it does not know is skipped.
fn sgr(h: *Handler, params: []const u16, colons: u32) void {
const s = &h.style;
if (params.len == 0) s.* = .{};
var i: usize = 0;
while (i < params.len) {
const p = params[i..];
const colon = colons >> @intCast(i) & 1 != 0;
i += 1;
// The colons on from p[1], short of the last parameter.
var subs: usize = 0;
while (i + subs < params.len - 1 and colons >> @intCast(i + subs) & 1 != 0) subs += 1;
if (colon and p[0] != 4 and p[0] != 38 and p[0] != 48 and p[0] != 58) {
while (colons >> @intCast(i) & 1 != 0) i += 1;
i += 1;
continue;
}
switch (p[0]) {
0 => s.* = .{},
1 => s.bold = true,
2 => s.dim = true,
3 => s.italic = true,
4 => if (!colon) {
s.ul = .single;
} else if (p.len >= 2) {
if (colons >> @intCast(i) & 1 != 0) {
i += subs + 1;
} else {
i += 1;
s.ul = if (p[1] <= 5) @enumFromInt(p[1]) else .single;
}
},
5, 6 => s.blink = true,
7 => s.reverse = true,
8 => s.invisible = true,
9 => s.strikethrough = true,
21 => s.ul = .double,
// SGR 22 ends faint as well as bold.
22 => {
s.bold = false;
s.dim = false;
},
23 => s.italic = false,
24 => s.ul = .off,
25 => s.blink = false,
27 => s.reverse = false,
28 => s.invisible = false,
29 => s.strikethrough = false,
30...37 => s.fg = .{ .index = @intCast(p[0] - 30) },
39 => s.fg = .default,
40...47 => s.bg = .{ .index = @intCast(p[0] - 40) },
49 => s.bg = .default,
90...97 => s.fg = .{ .index = @intCast(p[0] - 82) },
100...107 => s.bg = .{ .index = @intCast(p[0] - 92) },
// A colour: 5 and an index, or 2 and red, green and blue,
// after a colour space's slot when colons give one. A cell
// has no underline colour (58) to take.
38, 48, 58 => if (p.len >= 2) {
var color: ?Color = null;
if (p[1] == 5 and p.len >= 3) {
i += 2;
color = .{ .index = @truncate(p[2]) };
} else if (p[1] == 2 and p.len >= 5) {
if (colon and subs != 3 and subs != 4) {
i += subs + 1;
continue;
}
const at: usize = if (colon and subs == 4) 3 else 2;
i += at + 2;
color = .{ .rgb = .{ @truncate(p[at]), @truncate(p[at + 1]), @truncate(p[at + 2]) } };
}
if (color) |ink| switch (p[0]) {
38 => s.fg = ink,
48 => s.bg = ink,
else => {},
};
},
else => {},
}
}
}
// ponytail: bytes that are not UTF-8 reach here as U+FFFD (`parse`
// replaces them, as a terminal shows them); a raw-byte pass if a
// +Pager must keep them.
fn print(h: *Handler, cp: u21) void {
// A control a terminal acts on and never draws: DEL comes in a run
// of printed text, as a terminal's parse hands it on. A tab is
// HT's, printed here.
if ((cp < 0x20 and cp != '\t') or (cp >= 0x7f and cp < 0xa0)) return;
if (h.cr) {
h.text.shrinkRetainingCapacity(h.line_start);
h.cut(h.line_start);
h.last = h.line_start;
h.cr = false;
h.bs = false;
}
var buf: [4]u8 = undefined;
const n = std.unicode.utf8Encode(cp, &buf) catch std.unicode.utf8Encode(0xfffd, &buf) catch unreachable;
if (h.bs) {
// A man page's overstrike: a character struck over itself is
// bold, one struck with `_` (either way round) is underlined;
// any other is replaced, as on a terminal.
h.bs = false;
const prev = h.text.items[h.last..];
var struck = h.style;
if (std.mem.eql(u8, prev, buf[0..n])) {
struck.bold = true;
} else if (std.mem.eql(u8, prev, "_")) {
h.text.shrinkRetainingCapacity(h.last);
h.append(buf[0..n]);
struck.ul = .single;
} else if (cp == '_') {
struck.ul = .single;
} else {
h.text.shrinkRetainingCapacity(h.last);
h.cut(h.last);
h.last = h.text.items.len;
h.append(buf[0..n]);
h.span(h.last, h.style);
return;
}
h.cut(h.last);
h.span(h.last, struck);
return;
}
h.last = h.text.items.len;
h.append(buf[0..n]);
h.span(h.last, h.style);
}
/// A run of printed characters (most of a page): the first as `print`
/// takes it, a pending return or backspace and all, the rest straight
/// on, as one span. A 256 MiB page goes through here.
fn printRun(h: *Handler, cps: []const u32) void {
if (cps.len == 0) return;
h.print(@intCast(cps[0]));
const rest = cps[1..];
if (rest.len == 0) return;
var need: usize = 0;
for (rest) |cp| need += if (cp < 0x80) 1 else std.unicode.utf8CodepointSequenceLength(@intCast(cp)) catch 3;
h.text.ensureUnusedCapacity(h.gpa, need) catch {
h.failed = true;
return;
};
const from = h.text.items.len;
for (rest) |cp| {
if (cp < 0x20 or (cp >= 0x7f and cp < 0xa0)) continue;
h.last = h.text.items.len;
if (cp < 0x80) {
h.text.appendAssumeCapacity(@intCast(cp));
continue;
}
var buf: [4]u8 = undefined;
const n = std.unicode.utf8Encode(@intCast(cp), &buf) catch std.unicode.utf8Encode(0xfffd, &buf) catch unreachable;
h.text.appendSliceAssumeCapacity(buf[0..n]);
}
h.span(from, h.style);
}
fn append(h: *Handler, bytes: []const u8) void {
h.text.appendSlice(h.gpa, bytes) catch {
h.failed = true;
};
}
/// `from` to the text's end drawn in `style`: the span before it goes
/// on when it ends there in the same style.
fn span(h: *Handler, from: usize, style: CellStyle) void {
if (std.meta.eql(style, CellStyle{})) return;
const end: u32 = @intCast(h.text.items.len);
if (h.spans.items.len > 0) {
const tail = &h.spans.items[h.spans.items.len - 1];
if (tail.end == from and std.meta.eql(tail.style, style)) {
tail.end = end;
return;
}
}
if (h.spans.items.len >= max_spans) return;
h.spans.append(h.gpa, .{ .start = @intCast(from), .end = end, .style = style }) catch {
h.failed = true;
};
}
/// No span reaches past `at` any more: the text there is gone.
fn cut(h: *Handler, at: usize) void {
while (h.spans.items.len > 0) {
const tail = &h.spans.items[h.spans.items.len - 1];
if (tail.end <= at) return;
if (tail.start < at) {
tail.end = @intCast(at);
return;
}
_ = h.spans.pop();
}
}
};
// ---- tests ----
const testing = std.testing;
/// The text `raw` pages to, and its spans as `text[start..end]=style` lines.
fn expectPage(raw: []const u8, text: []const u8, spans: []const struct { []const u8, CellStyle }) !void {
const page = try parse(testing.allocator, raw);
defer page.deinit(testing.allocator);
try testing.expectEqualStrings(text, page.text);
for (spans, 0..) |want, i| {
if (i >= page.spans.len) {
std.debug.print("missing span {d}: \"{s}\"\n", .{ i, want[0] });
return error.TestExpectedEqual;
}
const got = page.spans[i];
try testing.expectEqualStrings(want[0], page.text[got.start..got.end]);
try testing.expectEqualDeep(want[1], got.style);
}
try testing.expectEqual(spans.len, page.spans.len);
}
test "git log --color: the commit line yellow, the rest plain" {
try expectPage(
"\x1b[33mcommit 0123abc\x1b[m\x1b[33m (\x1b[m\x1b[1;36mHEAD -> \x1b[m\x1b[1;32mmain\x1b[m\x1b[33m)\x1b[m\nAuthor: a \n\n subject\n",
"commit 0123abc (HEAD -> main)\nAuthor: a \n\n subject\n",
&.{
.{ "commit 0123abc (", .{ .fg = .{ .index = 3 } } },
.{ "HEAD -> ", .{ .fg = .{ .index = 6 }, .bold = true } },
.{ "main", .{ .fg = .{ .index = 2 }, .bold = true } },
.{ ")", .{ .fg = .{ .index = 3 } } },
},
);
}
test "ls --color=always: 01;34 directories and 01;32 executables, reset by 0 and none" {
try expectPage(
"\x1b[0m\x1b[01;34mdir\x1b[0m\n\x1b[01;32mrun.sh\x1b[0m\nplain.txt\n",
"dir\nrun.sh\nplain.txt\n",
&.{
.{ "dir", .{ .fg = .{ .index = 4 }, .bold = true } },
.{ "run.sh", .{ .fg = .{ .index = 2 }, .bold = true } },
},
);
}
test "grep --color=always: the match, the file name and the separator, and EL dropped" {
try expectPage(
"\x1b[35m\x1b[Ka.txt\x1b[m\x1b[K\x1b[36m\x1b[K:\x1b[m\x1b[Kone \x1b[01;31m\x1b[Kfoo\x1b[m\x1b[K two\n",
"a.txt:one foo two\n",
&.{
.{ "a.txt", .{ .fg = .{ .index = 5 } } },
.{ ":", .{ .fg = .{ .index = 6 } } },
.{ "foo", .{ .fg = .{ .index = 1 }, .bold = true } },
},
);
}
test "a man page's overstrike: bold and underline, and a stray backspace" {
try expectPage(
"N\x08NA\x08AM\x08ME\x08E _\x08u_\x08s x\x08_ a\x08b\x08\n",
"NAME us x b\n",
&.{
.{ "NAME", .{ .bold = true } },
.{ "us", .{ .ul = .single } },
.{ "x", .{ .ul = .single } },
},
);
}
test "truecolor, 256 colours, background, italic, faint, inverse and curly underline" {
try expectPage(
"\x1b[38;2;255;128;0mo\x1b[48;5;236mb\x1b[0;3;2mi\x1b[7mr\x1b[0;4:3mc\x1b[m.\n",
"obirc.\n",
&.{
.{ "o", .{ .fg = .{ .rgb = .{ 255, 128, 0 } } } },
.{ "b", .{ .fg = .{ .rgb = .{ 255, 128, 0 } }, .bg = .{ .index = 236 } } },
.{ "i", .{ .italic = true, .dim = true } },
.{ "r", .{ .italic = true, .dim = true, .reverse = true } },
.{ "c", .{ .ul = .curly } },
},
);
}
test "every other escape is dropped, and a stray or broken one loses no text after it" {
const cases = [_][2][]const u8{
.{ "a\x1b]8;;http://x\x1b\\link\x1b]8;;\x07b", "alinkb" },
.{ "\x1b]0;title\x07\x1b[2J\x1b[H\x1b[?1049hdone\n", "done\n" },
.{ "\x1b(Bdone", "done" },
.{ "plain\ttext\n", "plain\ttext\n" },
.{ "a\r\nb\r\n", "a\nb\n" },
.{ "10%\r50%\r\x1b[K100%\ndone\n", "100%\ndone\n" },
.{ "x\x00y\x07z\x0e\x0f\n", "xyz\n" },
// Controls that are dropped: CAN, FS, DEL and a C1 as UTF-8.
.{ "x\x18y\x1cz\x7f!\u{85}?\n", "xyz!?\n" },
// A broken CSI: its parameter bytes are eaten up to a final byte, as
// a terminal eats them; CAN abandons one, and the text goes on.
.{ "a\x1b[31;\x18b\x1b[m\n", "ab\n" },
// An unknown SGR changes nothing; an ESC at the very end is dropped.
.{ "\x1b[99mz\x1b", "z" },
};
for (cases) |c| try expectPage(c[0], c[1], &.{});
}
test "bytes that are not UTF-8 page as U+FFFD, the text around them kept" {
try expectPage("caf\xe9 \xff\xfe ok\n", "caf\u{fffd} \u{fffd}\u{fffd} ok\n", &.{});
}
test "a carriage return's rewrite takes the spans of what it rewrote with it" {
try expectPage("\x1b[31m10%\r\x1b[32m100%\x1b[m\n", "100%\n", &.{.{ "100%", .{ .fg = .{ .index = 2 } } }});
}
test "colon colours, an 8-bit CSI, REP, CHT, CNL, a colour short of its values and UTF-8 cut short" {
try expectPage("\x1b[38:2::1:2:3mx\x1b[48:5:7my\x1b[m", "xy", &.{
.{ "x", .{ .fg = .{ .rgb = .{ 1, 2, 3 } } } },
.{ "y", .{ .fg = .{ .rgb = .{ 1, 2, 3 } }, .bg = .{ .index = 7 } } },
});
// 0x9b in text is a stray byte, U+FFFD; in a sequence (after ESC
// here) it is CSI; as UTF-8 (C2 9B) it is a C1 control, dropped.
try expectPage("x\x9b\x1b\x9b31mr\xc2\x9b32mg", "x\u{fffd}r32mg", &.{.{ "r32mg", .{ .fg = .{ .index = 1 } } }});
// REP and CHT go on along the line; CNL returns, and `d` starts it over.
try expectPage("ab\x1b[3b\x1b[2Ic\n", "abbbb\t\tc\n", &.{});
try expectPage("ab\x1b[Ed\n", "d\n", &.{});
// 38;5 with no index: 38 is unknown and 5 is blink.
try expectPage("\x1b[38;5mx", "x", &.{.{ "x", .{ .blink = true } }});
try expectPage("ok\xe2\x82", "ok", &.{});
try expectPage("\xe2\x82x", "\u{fffd}x", &.{});
}