//! A program's output, paged into a +Pager (`pardes -`): the text it //! prints, and its SGR colours as spans over that text. It reads the bytes //! as a terminal does (ECMA-48's parse as ghostty-vt's table has it, UTF-8 //! with each ill-formed part one U+FFFD) and keeps what is printed, how SGR //! styles it and the controls that move along a line (CR, LF, BS, HT and //! CSI's CHT, REP and the return in CNL and CPL); every other sequence is //! read to its end and dropped (cursor motion, OSC titles and hyperlinks, //! DCS, APC, modes). The spans are display state of the pane only: no 9P //! file reads them (File.recolorSyntax draws them). const std = @import("std"); const pardes = @import("pardes.zig"); const CellStyle = @import("surface.zig").CellStyle; const Color = @import("surface.zig").Color; /// Bytes `start..end` of the text, drawn in `style`. A palette colour stays /// `.index`, resolved through the theme as it is drawn (terminal.palColor), /// and `.default` is the pane's own. pub const Span = struct { start: u32, end: u32, style: CellStyle }; /// ponytail: past this many spans the rest of a page is plain, rather than /// ~100 MB of spans for a 256 MiB coloured log; a packed style if it matters. pub const max_spans = 4 << 20; pub const Page = struct { text: []u8, spans: []Span, pub fn deinit(page: Page, gpa: std.mem.Allocator) void { gpa.free(page.text); gpa.free(page.spans); } }; /// `raw`, a program's output for a terminal, as text and its spans. Both /// are `gpa`'s. pub fn parse(gpa: std.mem.Allocator, raw: []const u8) !Page { var h: Handler = .{ .gpa = gpa }; errdefer h.text.deinit(gpa); errdefer h.spans.deinit(gpa); try h.text.ensureTotalCapacity(gpa, raw.len); // The printed code points since the last control, printed as one run. var run: [4096]u32 = undefined; var n: usize = 0; var state: State = .ground; var i: usize = 0; feed: while (i < raw.len) { const c = raw[i]; if (state != .ground or c == 0x1b) { h.printRun(run[0..n]); n = 0; state = h.step(state, c); i += 1; continue; } // Printable ASCII, most of a page, straight into the run. if (c >= 0x20 and c < 0x7f) { const end = @min(raw.len, i + run.len - n); while (i < end and raw[i] >= 0x20 and raw[i] < 0x7f) : (i += 1) { run[n] = raw[i]; n += 1; } if (n < run.len) continue; h.printRun(&run); n = 0; continue; } // UTF-8: an ill-formed part (a stray byte, or the longest start of // a sequence that does not go on) is one U+FFFD; one the input ends // inside may yet go on, and is not printed. var cp: u21 = c; var len: usize = 1; if (c >= 0x80) { cp = 0xfffd; if (c >= 0xc2 and c <= 0xf4) { const want: usize = if (c < 0xe0) 2 else if (c < 0xf0) 3 else 4; const lo: u8 = if (c == 0xe0) 0xa0 else if (c == 0xf0) 0x90 else 0x80; const hi: u8 = if (c == 0xed) 0x9f else if (c == 0xf4) 0x8f else 0xbf; var acc: u21 = c & (@as(u8, 0x7f) >> @intCast(want)); while (len < want) : (len += 1) { if (i + len == raw.len) break :feed; const b = raw[i + len]; if (b < (if (len == 1) lo else 0x80) or b > (if (len == 1) hi else 0xbf)) break; acc = acc << 6 | (b & 0x3f); } if (len == want) cp = acc; } } i += len; // A C1 control that came as UTF-8 is dropped, as xterm drops it. if (cp >= 0x20 and (cp < 0x80 or cp > 0x9f)) { run[n] = cp; n += 1; if (n < run.len) continue; } h.printRun(run[0..n]); n = 0; if (cp < 0x20) h.execute(@intCast(cp)); } h.printRun(run[0..n]); if (h.failed) return error.OutOfMemory; const text = try h.text.toOwnedSlice(gpa); errdefer gpa.free(text); return .{ .text = text, .spans = try h.spans.toOwnedSlice(gpa) }; } /// Where the parse is in a sequence: ECMA-48's states, the DCS's data and /// its ignored tail one (`dcs_body`) and SOS, PM and APC one (`apc`). const State = enum { ground, esc, esc_inter, csi_entry, csi_param, csi_inter, csi_ignore, osc, dcs_entry, dcs_param, dcs_inter, dcs_body, apc }; const Handler = struct { gpa: std.mem.Allocator, text: std.ArrayList(u8) = .empty, spans: std.ArrayList(Span) = .empty, /// What SGR has set so far. style: CellStyle = .{}, /// Where the line being printed starts, and its last character. line_start: usize = 0, last: usize = 0, /// A carriage return not yet followed by a newline: the line starts /// over at the next print, as on a terminal, so a progress line keeps /// its last state. cr: bool = false, /// A backspace: the next print strikes over the character before it. bs: bool = false, failed: bool = false, /// The control sequence being read: its parameters as ghostty-vt keeps /// them (24 at most, each saturating at 65535, a bit in `colons` for /// each one a colon ends), and whether it has an intermediate or a /// private marker, which no sequence kept here takes. csi: struct { params: [24]u16 = undefined, n: u8 = 0, acc: u16 = 0, digits: bool = false, colons: u32 = 0, inter: bool = false, } = .{}, /// The state after byte `c` in `state` (not ground's printing, which /// `parse` decodes), acting on the controls and sequences a page keeps. fn step(h: *Handler, state: State, c: u8) State { switch (c) { 0x18, 0x1a => return .ground, 0x1b => { h.csi = .{}; return .esc; }, // A C1 control byte, in all but an OSC's or a DCS's data. 0x80...0x9f => return if (state == .osc or state == .dcs_body) state else switch (c) { 0x90 => .dcs_entry, 0x9b => csi: { h.csi = .{}; break :csi .csi_entry; }, 0x9d => .osc, 0x98, 0x9e, 0x9f => .apc, else => .ground, }, 0xa0...0xff => return state, else => {}, } switch (state) { .osc => return if (c == 0x07) .ground else .osc, .apc, .dcs_body => return state, .dcs_entry, .dcs_param, .dcs_inter => return switch (c) { 0...0x1f, 0x7f => state, 0x20...0x2f => .dcs_inter, '0'...'9', ';' => if (state == .dcs_inter) .dcs_body else .dcs_param, '<'...'?' => if (state == .dcs_entry) .dcs_param else .dcs_body, else => .dcs_body, }, else => {}, } if (c < 0x20) { h.execute(c); return state; } if (c == 0x7f) return state; switch (state) { .esc => return switch (c) { '[' => .csi_entry, ']' => .osc, 'P' => .dcs_entry, 'X', '^', '_' => .apc, 0x20...0x2f => .esc_inter, else => .ground, }, .esc_inter => return if (c < 0x30) state else .ground, .csi_ignore => return if (c < 0x40) state else .ground, else => {}, } const p = &h.csi; switch (c) { 0x20...0x2f => { p.inter = true; return .csi_inter; }, '0'...'9', ':', ';' => { if (state == .csi_inter or (c == ':' and state == .csi_entry)) return .csi_ignore; if (p.n == p.params.len) return .csi_param; if (c <= '9') { p.acc = p.acc *| 10 +| (c - '0'); p.digits = true; return .csi_param; } p.params[p.n] = p.acc; if (c == ':') p.colons |= @as(u32, 1) << @intCast(p.n); p.n += 1; p.acc = 0; p.digits = false; return .csi_param; }, '<'...'?' => { if (state != .csi_entry) return .csi_ignore; p.inter = true; return .csi_param; }, else => { h.dispatch(c); return .ground; }, } } /// A C0 control: the line feeds (LF, VT and FF), CR, BS and HT. fn execute(h: *Handler, c: u8) void { switch (c) { '\n', 0x0b, 0x0c => { h.append("\n"); h.line_start = h.text.items.len; h.last = h.line_start; h.cr = false; h.bs = false; }, '\r' => h.cr = true, 0x08 => h.bs = h.text.items.len > h.line_start, '\t' => h.print('\t'), else => {}, } } /// The control sequence ending in `f`: SGR, and of the rest what moves /// along a line, dropped as ghostty-vt drops a malformed one. fn dispatch(h: *Handler, f: u8) void { const p = &h.csi; if (p.n == p.params.len) return; if (p.digits) { p.params[p.n] = p.acc; p.n += 1; } if (p.inter or (f != 'm' and p.colons != 0)) return; if (f == 'm') return h.sgr(p.params[0..p.n], p.colons); if (p.n > 1) return; const count: u16 = if (p.n == 0) 1 else p.params[0]; switch (f) { // CNL and CPL: down or up, which a page has not, and a return. 'E', 'F' => h.cr = true, // CHT; and REP, the last character again, as a terminal draws // it: a terminal stops at its edge, and this at a line's worth. 'I' => for (0..@min(count, 64)) |_| h.print('\t'), 'b' => if (h.text.items.len > h.last) { const cp = std.unicode.utf8Decode(h.text.items[h.last..]) catch return; for (0..@min(count, 1024)) |_| h.print(cp); }, else => {}, } } /// SGR's parameters into the style, read as ghostty-vt's sgr.Parser /// reads them: a colour short of its values leaves them to be read as /// SGRs of their own, and a colon list it does not know is skipped. fn sgr(h: *Handler, params: []const u16, colons: u32) void { const s = &h.style; if (params.len == 0) s.* = .{}; var i: usize = 0; while (i < params.len) { const p = params[i..]; const colon = colons >> @intCast(i) & 1 != 0; i += 1; // The colons on from p[1], short of the last parameter. var subs: usize = 0; while (i + subs < params.len - 1 and colons >> @intCast(i + subs) & 1 != 0) subs += 1; if (colon and p[0] != 4 and p[0] != 38 and p[0] != 48 and p[0] != 58) { while (colons >> @intCast(i) & 1 != 0) i += 1; i += 1; continue; } switch (p[0]) { 0 => s.* = .{}, 1 => s.bold = true, 2 => s.dim = true, 3 => s.italic = true, 4 => if (!colon) { s.ul = .single; } else if (p.len >= 2) { if (colons >> @intCast(i) & 1 != 0) { i += subs + 1; } else { i += 1; s.ul = if (p[1] <= 5) @enumFromInt(p[1]) else .single; } }, 5, 6 => s.blink = true, 7 => s.reverse = true, 8 => s.invisible = true, 9 => s.strikethrough = true, 21 => s.ul = .double, // SGR 22 ends faint as well as bold. 22 => { s.bold = false; s.dim = false; }, 23 => s.italic = false, 24 => s.ul = .off, 25 => s.blink = false, 27 => s.reverse = false, 28 => s.invisible = false, 29 => s.strikethrough = false, 30...37 => s.fg = .{ .index = @intCast(p[0] - 30) }, 39 => s.fg = .default, 40...47 => s.bg = .{ .index = @intCast(p[0] - 40) }, 49 => s.bg = .default, 90...97 => s.fg = .{ .index = @intCast(p[0] - 82) }, 100...107 => s.bg = .{ .index = @intCast(p[0] - 92) }, // A colour: 5 and an index, or 2 and red, green and blue, // after a colour space's slot when colons give one. A cell // has no underline colour (58) to take. 38, 48, 58 => if (p.len >= 2) { var color: ?Color = null; if (p[1] == 5 and p.len >= 3) { i += 2; color = .{ .index = @truncate(p[2]) }; } else if (p[1] == 2 and p.len >= 5) { if (colon and subs != 3 and subs != 4) { i += subs + 1; continue; } const at: usize = if (colon and subs == 4) 3 else 2; i += at + 2; color = .{ .rgb = .{ @truncate(p[at]), @truncate(p[at + 1]), @truncate(p[at + 2]) } }; } if (color) |ink| switch (p[0]) { 38 => s.fg = ink, 48 => s.bg = ink, else => {}, }; }, else => {}, } } } // ponytail: bytes that are not UTF-8 reach here as U+FFFD (`parse` // replaces them, as a terminal shows them); a raw-byte pass if a // +Pager must keep them. fn print(h: *Handler, cp: u21) void { // A control a terminal acts on and never draws: DEL comes in a run // of printed text, as a terminal's parse hands it on. A tab is // HT's, printed here. if ((cp < 0x20 and cp != '\t') or (cp >= 0x7f and cp < 0xa0)) return; if (h.cr) { h.text.shrinkRetainingCapacity(h.line_start); h.cut(h.line_start); h.last = h.line_start; h.cr = false; h.bs = false; } var buf: [4]u8 = undefined; const n = std.unicode.utf8Encode(cp, &buf) catch std.unicode.utf8Encode(0xfffd, &buf) catch unreachable; if (h.bs) { // A man page's overstrike: a character struck over itself is // bold, one struck with `_` (either way round) is underlined; // any other is replaced, as on a terminal. h.bs = false; const prev = h.text.items[h.last..]; var struck = h.style; if (std.mem.eql(u8, prev, buf[0..n])) { struck.bold = true; } else if (std.mem.eql(u8, prev, "_")) { h.text.shrinkRetainingCapacity(h.last); h.append(buf[0..n]); struck.ul = .single; } else if (cp == '_') { struck.ul = .single; } else { h.text.shrinkRetainingCapacity(h.last); h.cut(h.last); h.last = h.text.items.len; h.append(buf[0..n]); h.span(h.last, h.style); return; } h.cut(h.last); h.span(h.last, struck); return; } h.last = h.text.items.len; h.append(buf[0..n]); h.span(h.last, h.style); } /// A run of printed characters (most of a page): the first as `print` /// takes it, a pending return or backspace and all, the rest straight /// on, as one span. A 256 MiB page goes through here. fn printRun(h: *Handler, cps: []const u32) void { if (cps.len == 0) return; h.print(@intCast(cps[0])); const rest = cps[1..]; if (rest.len == 0) return; var need: usize = 0; for (rest) |cp| need += if (cp < 0x80) 1 else std.unicode.utf8CodepointSequenceLength(@intCast(cp)) catch 3; h.text.ensureUnusedCapacity(h.gpa, need) catch { h.failed = true; return; }; const from = h.text.items.len; for (rest) |cp| { if (cp < 0x20 or (cp >= 0x7f and cp < 0xa0)) continue; h.last = h.text.items.len; if (cp < 0x80) { h.text.appendAssumeCapacity(@intCast(cp)); continue; } var buf: [4]u8 = undefined; const n = std.unicode.utf8Encode(@intCast(cp), &buf) catch std.unicode.utf8Encode(0xfffd, &buf) catch unreachable; h.text.appendSliceAssumeCapacity(buf[0..n]); } h.span(from, h.style); } fn append(h: *Handler, bytes: []const u8) void { h.text.appendSlice(h.gpa, bytes) catch { h.failed = true; }; } /// `from` to the text's end drawn in `style`: the span before it goes /// on when it ends there in the same style. fn span(h: *Handler, from: usize, style: CellStyle) void { if (std.meta.eql(style, CellStyle{})) return; const end: u32 = @intCast(h.text.items.len); if (h.spans.items.len > 0) { const tail = &h.spans.items[h.spans.items.len - 1]; if (tail.end == from and std.meta.eql(tail.style, style)) { tail.end = end; return; } } if (h.spans.items.len >= max_spans) return; h.spans.append(h.gpa, .{ .start = @intCast(from), .end = end, .style = style }) catch { h.failed = true; }; } /// No span reaches past `at` any more: the text there is gone. fn cut(h: *Handler, at: usize) void { while (h.spans.items.len > 0) { const tail = &h.spans.items[h.spans.items.len - 1]; if (tail.end <= at) return; if (tail.start < at) { tail.end = @intCast(at); return; } _ = h.spans.pop(); } } }; // ---- tests ---- const testing = std.testing; /// The text `raw` pages to, and its spans as `text[start..end]=style` lines. fn expectPage(raw: []const u8, text: []const u8, spans: []const struct { []const u8, CellStyle }) !void { const page = try parse(testing.allocator, raw); defer page.deinit(testing.allocator); try testing.expectEqualStrings(text, page.text); for (spans, 0..) |want, i| { if (i >= page.spans.len) { std.debug.print("missing span {d}: \"{s}\"\n", .{ i, want[0] }); return error.TestExpectedEqual; } const got = page.spans[i]; try testing.expectEqualStrings(want[0], page.text[got.start..got.end]); try testing.expectEqualDeep(want[1], got.style); } try testing.expectEqual(spans.len, page.spans.len); } test "git log --color: the commit line yellow, the rest plain" { try expectPage( "\x1b[33mcommit 0123abc\x1b[m\x1b[33m (\x1b[m\x1b[1;36mHEAD -> \x1b[m\x1b[1;32mmain\x1b[m\x1b[33m)\x1b[m\nAuthor: a \n\n subject\n", "commit 0123abc (HEAD -> main)\nAuthor: a \n\n subject\n", &.{ .{ "commit 0123abc (", .{ .fg = .{ .index = 3 } } }, .{ "HEAD -> ", .{ .fg = .{ .index = 6 }, .bold = true } }, .{ "main", .{ .fg = .{ .index = 2 }, .bold = true } }, .{ ")", .{ .fg = .{ .index = 3 } } }, }, ); } test "ls --color=always: 01;34 directories and 01;32 executables, reset by 0 and none" { try expectPage( "\x1b[0m\x1b[01;34mdir\x1b[0m\n\x1b[01;32mrun.sh\x1b[0m\nplain.txt\n", "dir\nrun.sh\nplain.txt\n", &.{ .{ "dir", .{ .fg = .{ .index = 4 }, .bold = true } }, .{ "run.sh", .{ .fg = .{ .index = 2 }, .bold = true } }, }, ); } test "grep --color=always: the match, the file name and the separator, and EL dropped" { try expectPage( "\x1b[35m\x1b[Ka.txt\x1b[m\x1b[K\x1b[36m\x1b[K:\x1b[m\x1b[Kone \x1b[01;31m\x1b[Kfoo\x1b[m\x1b[K two\n", "a.txt:one foo two\n", &.{ .{ "a.txt", .{ .fg = .{ .index = 5 } } }, .{ ":", .{ .fg = .{ .index = 6 } } }, .{ "foo", .{ .fg = .{ .index = 1 }, .bold = true } }, }, ); } test "a man page's overstrike: bold and underline, and a stray backspace" { try expectPage( "N\x08NA\x08AM\x08ME\x08E _\x08u_\x08s x\x08_ a\x08b\x08\n", "NAME us x b\n", &.{ .{ "NAME", .{ .bold = true } }, .{ "us", .{ .ul = .single } }, .{ "x", .{ .ul = .single } }, }, ); } test "truecolor, 256 colours, background, italic, faint, inverse and curly underline" { try expectPage( "\x1b[38;2;255;128;0mo\x1b[48;5;236mb\x1b[0;3;2mi\x1b[7mr\x1b[0;4:3mc\x1b[m.\n", "obirc.\n", &.{ .{ "o", .{ .fg = .{ .rgb = .{ 255, 128, 0 } } } }, .{ "b", .{ .fg = .{ .rgb = .{ 255, 128, 0 } }, .bg = .{ .index = 236 } } }, .{ "i", .{ .italic = true, .dim = true } }, .{ "r", .{ .italic = true, .dim = true, .reverse = true } }, .{ "c", .{ .ul = .curly } }, }, ); } test "every other escape is dropped, and a stray or broken one loses no text after it" { const cases = [_][2][]const u8{ .{ "a\x1b]8;;http://x\x1b\\link\x1b]8;;\x07b", "alinkb" }, .{ "\x1b]0;title\x07\x1b[2J\x1b[H\x1b[?1049hdone\n", "done\n" }, .{ "\x1b(Bdone", "done" }, .{ "plain\ttext\n", "plain\ttext\n" }, .{ "a\r\nb\r\n", "a\nb\n" }, .{ "10%\r50%\r\x1b[K100%\ndone\n", "100%\ndone\n" }, .{ "x\x00y\x07z\x0e\x0f\n", "xyz\n" }, // Controls that are dropped: CAN, FS, DEL and a C1 as UTF-8. .{ "x\x18y\x1cz\x7f!\u{85}?\n", "xyz!?\n" }, // A broken CSI: its parameter bytes are eaten up to a final byte, as // a terminal eats them; CAN abandons one, and the text goes on. .{ "a\x1b[31;\x18b\x1b[m\n", "ab\n" }, // An unknown SGR changes nothing; an ESC at the very end is dropped. .{ "\x1b[99mz\x1b", "z" }, }; for (cases) |c| try expectPage(c[0], c[1], &.{}); } test "bytes that are not UTF-8 page as U+FFFD, the text around them kept" { try expectPage("caf\xe9 \xff\xfe ok\n", "caf\u{fffd} \u{fffd}\u{fffd} ok\n", &.{}); } test "a carriage return's rewrite takes the spans of what it rewrote with it" { try expectPage("\x1b[31m10%\r\x1b[32m100%\x1b[m\n", "100%\n", &.{.{ "100%", .{ .fg = .{ .index = 2 } } }}); } test "colon colours, an 8-bit CSI, REP, CHT, CNL, a colour short of its values and UTF-8 cut short" { try expectPage("\x1b[38:2::1:2:3mx\x1b[48:5:7my\x1b[m", "xy", &.{ .{ "x", .{ .fg = .{ .rgb = .{ 1, 2, 3 } } } }, .{ "y", .{ .fg = .{ .rgb = .{ 1, 2, 3 } }, .bg = .{ .index = 7 } } }, }); // 0x9b in text is a stray byte, U+FFFD; in a sequence (after ESC // here) it is CSI; as UTF-8 (C2 9B) it is a C1 control, dropped. try expectPage("x\x9b\x1b\x9b31mr\xc2\x9b32mg", "x\u{fffd}r32mg", &.{.{ "r32mg", .{ .fg = .{ .index = 1 } } }}); // REP and CHT go on along the line; CNL returns, and `d` starts it over. try expectPage("ab\x1b[3b\x1b[2Ic\n", "abbbb\t\tc\n", &.{}); try expectPage("ab\x1b[Ed\n", "d\n", &.{}); // 38;5 with no index: 38 is unknown and 5 is blink. try expectPage("\x1b[38;5mx", "x", &.{.{ "x", .{ .blink = true } }}); try expectPage("ok\xe2\x82", "ok", &.{}); try expectPage("\xe2\x82x", "\u{fffd}x", &.{}); }