A terminal pane draws the kitty graphics its program places, so yazi's image and PDF previews show

yazi asks the pane XTVERSION, and ghostty-vt answers "libghostty": yazi
takes it for Ghostty and sends its previews by the kitty graphics protocol
with Unicode placeholders (a=T,U=1, then U+10EEEE cells). ghostty-vt stored
the images, but nothing drew them: the placeholders showed as tofu in the
GUI and as raw codepoints in the tty, in every build since terminal panes
answered queries (vkznsvzs, 2026-07-01). Before that no query was answered
and yazi fell back to chafa's text art.

Each frame, TerminalImages.zig reads the active screen's images and
placements (virtual ones through their placeholder runs, and ones placed on
a cell), cuts each into the cell rows it covers, maps those rows through
the body's own row walk, clips them to the pane, and hands them on as
ImagePlaces with exact geometry (NativePlacement.exact). The GUI draws
them as textures; the tty passes them to an outer terminal with kitty
graphics as one placement per row, its source cropped to whole cells.
Placeholder cells draw no text. Scrolling, a delete, a resize, the
alternate screen and a closed pane all fall out of reading the emulator's
state every frame.

A pane now knows its size in pixels: CSI 14/16/18 t are answered, the
emulator's width_px/height_px follow the shell's cells, and the pty's
winsize carries pixels (host_io.ptyWinsize), so yazi and a nested pardes
size their images right. ghostty-vt's image store is 64 MiB per screen
(its library default, 10 MB, refused a large preview), PNG transmissions
are decoded with stb_image, and RGB images are converted only while shown.

test/yazi_preview.py runs yazi in a hidden window on a PNG and a PDF and
checks the captures, also through a tty pardes nested in the pane.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

1 files changed, 247 insertions, 0 deletions

diff --git a/src/TerminalImages.zig b/src/TerminalImages.zig
new file mode 100644
index 00000000..773297b2
--- /dev/null
+++ b/src/TerminalImages.zig
@@ -0,0 +1,247 @@
+//! A terminal pane's kitty graphics: the images its program sent (yazi's
+//! previews, `kitten icat`) drawn where it placed them. ghostty-vt keeps
+//! the images and placements; this reads them each frame, cuts each one
+//! into the cell rows it covers, clipped to the rows the pane shows, and
+//! hands them on as pixels (`ImagePlace`, exact geometry) to a shell that
+//! draws them.
+//! Both kinds of placement are drawn: one on a cell (`a=T`/`a=p`), which
+//! moves with the text it was put on, and a virtual one (`U=1`) shown
+//! through the U+10EEEE placeholder cells the program prints.
+const std = @import("std");
+const ghostty_vt = @import("ghostty-vt");
+const pardes = @import("pardes.zig");
+const image = @import("image.zig");
+
+const TerminalImages = @This();
+const gfx = ghostty_vt.kitty.graphics;
+const PixelRect = image.PixelRect;
+
+/// The bytes ghostty-vt may hold for one screen's images (it evicts the
+/// oldest past this): a 4K picture in RGBA, with room for a second.
+pub const storage_limit = 64 * 1024 * 1024;
+
+/// Pixels in RGBA for images ghostty-vt holds in another format (RGB,
+/// grey): only for images on screen, dropped the frame they are not.
+converted: std.AutoHashMapUnmanaged(u32, Converted) = .empty,
+
+const Converted = struct { generation: u64, rgba: []u8, used: bool = true };
+
+/// One image's rectangle in the pane, in pixels from the viewport's top
+/// left corner (above or left of it is negative).
+const Rect = struct { x: i64, y: i64, w: i64, h: i64 };
+
+pub fn deinit(ti: *TerminalImages, gpa: std.mem.Allocator) void {
+    var conv = ti.converted.valueIterator();
+    while (conv.next()) |c| gpa.free(c.rgba);
+    ti.converted.deinit(gpa);
+}
+
+/// Accept PNG (`f=100`) as well as raw pixels: ghostty-vt built as a
+/// library decodes none itself. stb_image does it, as for image panes.
+pub fn installPngDecoder() void {
+    ghostty_vt.sys.decode_png = &decodePng;
+}
+
+fn decodePng(gpa: std.mem.Allocator, bytes: []const u8) ghostty_vt.sys.DecodeError!ghostty_vt.sys.Image {
+    const decoded = image.decode(gpa, bytes) orelse return error.InvalidData;
+    return .{ .width = @intCast(decoded.w), .height = @intCast(decoded.h), .data = decoded.rgba };
+}
+
+/// Draw the active screen's images. `rows_of[v]` is the body row the
+/// viewport's row `v` is shown on, if it is shown; `cols` how many of its
+/// columns the body shows, from `tx`.
+pub fn draw(
+    ti: *TerminalImages,
+    p: *pardes.Pardes,
+    s: *pardes.Surface,
+    vt: *ghostty_vt.Terminal,
+    pane_id: u8,
+    serial: u32,
+    tx: u16,
+    body_y: u16,
+    cols: u16,
+    rows_of: []const ?u16,
+) void {
+    var conv = ti.converted.valueIterator();
+    while (conv.next()) |c| c.used = false;
+    defer ti.prune(p.gpa);
+
+    const screen = vt.screens.active;
+    const storage = &screen.kitty_images;
+    const pages = &screen.pages;
+    const top = pages.getTopLeft(.viewport);
+    const bottom = pages.getBottomRight(.viewport) orelse return;
+    const cw: u32 = @max(1, p.cell_pixels.w);
+    const ch: u32 = @max(1, p.cell_pixels.h);
+    const ctx: Ctx = .{ .ti = ti, .p = p, .pane = pane_id, .serial = serial, .tx = tx, .body_y = body_y, .cols = cols, .rows_of = rows_of, .cw = cw, .ch = ch };
+
+    // Virtual placements: shown through the placeholder cells, one run of
+    // them on a row at a time. The placeholders themselves are not text.
+    var runs = gfx.unicode.placementIterator(top, bottom);
+    while (runs.next()) |run| {
+        const at = pages.pointFromPin(.viewport, run.pin) orelse continue;
+        const v = at.viewport.y;
+        if (v >= rows_of.len) continue;
+        const row = rows_of[v] orelse continue;
+        for (@as(usize, at.viewport.x)..@as(usize, at.viewport.x) + run.width) |x| {
+            if (x >= cols) break;
+            const cell = s.at(tx + @as(u16, @intCast(x)), body_y + row);
+            cell.text[0] = ' ';
+            cell.len = 1;
+        }
+        const img = storage.imageById(run.image_id) orelse continue;
+        const rp = run.renderPlacement(storage, &img, cw, ch) catch continue;
+        if (rp.dest_width == 0 or rp.dest_height == 0 or rp.source_width == 0 or rp.source_height == 0) continue;
+        ctx.place(img, .{ .x = rp.source_x, .y = rp.source_y, .w = rp.source_width, .h = rp.source_height }, .{
+            .x = @as(i64, at.viewport.x) * cw + rp.offset_x,
+            .y = @as(i64, v) * ch + rp.offset_y,
+            .w = rp.dest_width,
+            .h = rp.dest_height,
+        });
+    }
+
+    // Placements on a cell: they move with the text they were put on, and
+    // go where it goes, above the viewport or off the screen.
+    const origin = pages.pointFromPin(.screen, top) orelse return;
+    var it = storage.placements.iterator();
+    while (it.next()) |entry| {
+        const placed = entry.value_ptr.*;
+        const pin = switch (placed.location) {
+            .pin => |pin| pin,
+            .virtual => continue,
+        };
+        const img = storage.images.get(entry.key_ptr.image_id) orelse continue;
+        const at = pages.pointFromPin(.screen, pin.*) orelse continue;
+        const size = placed.pixelSize(img, vt);
+        const sx = @min(placed.source_x, img.width);
+        const sy = @min(placed.source_y, img.height);
+        const sw = if (placed.source_width > 0) @min(placed.source_width, img.width - sx) else img.width - sx;
+        const sh = if (placed.source_height > 0) @min(placed.source_height, img.height - sy) else img.height - sy;
+        if (size.width == 0 or size.height == 0 or sw == 0 or sh == 0) continue;
+        ctx.place(img, .{ .x = sx, .y = sy, .w = sw, .h = sh }, .{
+            .x = @as(i64, at.screen.x) * cw + placed.x_offset,
+            .y = (@as(i64, at.screen.y) - @as(i64, origin.screen.y)) * ch + placed.y_offset,
+            .w = size.width,
+            .h = size.height,
+        });
+    }
+}
+
+const Ctx = struct {
+    ti: *TerminalImages,
+    p: *pardes.Pardes,
+    pane: u8,
+    serial: u32,
+    tx: u16,
+    body_y: u16,
+    cols: u16,
+    rows_of: []const ?u16,
+    cw: u32,
+    ch: u32,
+
+    /// Draw `src` of `img` over `dst`, one cell row of it at a time: a row
+    /// is shown, or not, or shown elsewhere, as the body lays it out.
+    fn place(ctx: Ctx, img: gfx.Image, src: PixelRect, dst: Rect) void {
+        const cw: i64 = ctx.cw;
+        const ch: i64 = ctx.ch;
+        const x0 = @max(dst.x, 0);
+        const x1 = @min(dst.x + dst.w, @as(i64, ctx.cols) * cw);
+        if (x1 <= x0) return;
+        const rgba = ctx.ti.pixelsOf(ctx.p.gpa, img) orelse return;
+        const first_row = @divFloor(@max(dst.y, 0), ch);
+        const last_row = @min(@divFloor(dst.y + dst.h - 1, ch), @as(i64, @intCast(ctx.rows_of.len)) - 1);
+        var v = first_row;
+        while (v <= last_row) : (v += 1) {
+            const row = ctx.rows_of[@intCast(v)] orelse continue;
+            const y0 = @max(dst.y, v * ch);
+            const y1 = @min(dst.y + dst.h, (v + 1) * ch);
+            if (y1 <= y0) continue;
+            // The source pixels under this piece, by the same scale.
+            const piece: PixelRect = .{
+                .x = src.x + @as(u32, @intCast(@divFloor((x0 - dst.x) * src.w, dst.w))),
+                .y = src.y + @as(u32, @intCast(@divFloor((y0 - dst.y) * src.h, dst.h))),
+                .w = @max(1, @as(u32, @intCast(@divFloor((x1 - x0) * src.w, dst.w)))),
+                .h = @max(1, @as(u32, @intCast(@divFloor((y1 - y0) * src.h, dst.h)))),
+            };
+            const c0: u16 = @intCast(@divFloor(x0, cw));
+            const c1: u16 = @intCast(@divFloor(x1 + cw - 1, cw));
+            _ = ctx.p.appendImagePlace(.{
+                .pane = ctx.pane,
+                .serial = ctx.serial,
+                .native = .{
+                    .page = img.id,
+                    .revision = @truncate(img.generation),
+                    .exact = true,
+                    .geometry = .{ .src = piece, .dst = .{
+                        .x = @intCast(x0 - @as(i64, c0) * cw),
+                        .y = @intCast(y0 - v * ch),
+                        .w = @intCast(x1 - x0),
+                        .h = @intCast(y1 - y0),
+                    } },
+                },
+                .x = ctx.tx + c0,
+                .y = ctx.body_y + row,
+                .w = c1 - c0,
+                .h = 1,
+                .rgba = rgba,
+                .iw = img.width,
+                .ih = img.height,
+            });
+        }
+    }
+};
+
+/// The image's pixels as RGBA: its own bytes when it is held so.
+fn pixelsOf(ti: *TerminalImages, gpa: std.mem.Allocator, img: gfx.Image) ?[]const u8 {
+    const n = @as(usize, img.width) * img.height;
+    if (n == 0) return null;
+    if (img.format == .rgba) return if (img.data.len == n * 4) img.data else null;
+    const gop = ti.converted.getOrPut(gpa, img.id) catch return null;
+    if (gop.found_existing) {
+        if (gop.value_ptr.generation == img.generation) {
+            gop.value_ptr.used = true;
+            return gop.value_ptr.rgba;
+        }
+        gpa.free(gop.value_ptr.rgba);
+        _ = ti.converted.remove(img.id);
+        return ti.pixelsOf(gpa, img);
+    }
+    const out = convert(gpa, img, n) orelse {
+        _ = ti.converted.remove(img.id);
+        return null;
+    };
+    gop.value_ptr.* = .{ .generation = img.generation, .rgba = out };
+    return out;
+}
+
+fn convert(gpa: std.mem.Allocator, img: gfx.Image, n: usize) ?[]u8 {
+    const depth: usize = switch (img.format) {
+        .rgb => 3,
+        .gray_alpha => 2,
+        .gray => 1,
+        else => return null,
+    };
+    if (img.data.len != n * depth) return null;
+    const out = gpa.alloc(u8, n * 4) catch return null;
+    for (0..n) |i| {
+        const px = img.data[i * depth ..][0..depth];
+        out[i * 4 ..][0..4].* = switch (depth) {
+            3 => .{ px[0], px[1], px[2], 255 },
+            2 => .{ px[0], px[0], px[0], px[1] },
+            else => .{ px[0], px[0], px[0], 255 },
+        };
+    }
+    return out;
+}
+
+/// Drop what this frame did not draw: the caches hold only what is shown.
+fn prune(ti: *TerminalImages, gpa: std.mem.Allocator) void {
+    while (true) {
+        var conv = ti.converted.iterator();
+        const stale = while (conv.next()) |entry| {
+            if (!entry.value_ptr.used) break entry.key_ptr.*;
+        } else break;
+        gpa.free(ti.converted.get(stale).?.rgba);
+        _ = ti.converted.remove(stale);
+    }
+}