Shader files compile again on save; an attached GUI runs the session's post chain

shader_build.zig takes the Shadertoy compile out of the GUI's Post: the
process that holds the core compiles each chain file behind the prefix on a
thread, watches its directory (file_watch's new shader slots), and on a
change reads and hashes each file, compiling only one whose bytes moved. A
failed compile keeps the last good SPIR-V and is said once; the same bytes
are never compiled or reported twice. A file is read with std, not fs.zig's
readFile, whose turn hand-off a worker thread does not hold.

A detached session compiles for its frontends and sends them the chain
(wire post: each pass's scene and level or the file's SPIR-V, and
ShaderAnimation) on attach and on every change, so an attached GUI runs the
same passes, levels and animation as a local one while still reading no disk
and running no program. The attached GUI describes its frames to the chain
from the session's chrome and redraws an animating chain on its own.

2 files changed, 73 insertions, 219 deletions

diff --git a/src/gui/Post.zig b/src/gui/Post.zig
index 61ebc2cd..06310b7f 100644
--- a/src/gui/Post.zig
+++ b/src/gui/Post.zig
@@ -1,10 +1,10 @@
 //! The post chain (docs/render-pipeline.md §6): Shadertoy passes over the
 //! finished frame, each reading the one before it (the frame first) and the
 //! last one writing the window. A pass is the bundled Crt (shaders/post/,
-//! compiled with the build) or a user's file (Shader <path>),
-//! compiled when it joins the chain by the build's own glslc invocation on a
-//! thread of its own, never on the frame path. A file whose compile fails
-//! keeps its last good pipeline, and glslc missing is said once.
+//! compiled with the build) or a user's file (Shader <path>), whose SPIR-V
+//! comes from shader_build.zig: compiled by the process that holds the core,
+//! this one or the detached session this GUI is attached to. A file whose
+//! compile fails keeps its last good pipeline.
 //!
 //! The uniforms are ghostty's (Uniforms), so a shader written for ghostty runs
 //! here. Their clock is the shell's and moves only when the chain draws:
@@ -14,7 +14,6 @@ const std = @import("std");
 const libc = std.c;
 const ghostty_vt = @import("ghostty-vt");
 const pardes = @import("../pardes.zig");
-const filesystem = @import("../fs.zig");
 const gui = @import("gui.zig");
 const c = gui.c;
 
@@ -25,6 +24,7 @@ const Pipeline = c.SDL_GPUGraphicsPipeline;
 pub const max = Chain.max;
 
 const prefix = @embedFile("post-prefix.glsl");
+const Built = @import("../shader_build.zig").Pass;
 const vert_spv = @embedFile("post.vert.spv");
 const bundled_spv = std.EnumArray(Scene, []const u8).init(.{
     .crt = @embedFile("post-crt.frag.spv"),
@@ -92,28 +92,15 @@ pub const Pass = struct {
     /// A user's file, owned; empty for a bundled pass.
     path: []u8 = &.{},
     pipeline: ?*Pipeline = null,
-};
-
-/// One compile of a chain's new files, on its own thread.
-const Job = struct {
-    const Result = union(enum) { none, spirv: []u8, failed: []u8, missing };
-    io: std.Io,
-    gpa: std.mem.Allocator,
-    paths: [max][]u8 = @splat(&.{}),
-    results: [max]Result = @splat(.none),
-    len: usize = 0,
-    done: std.atomic.Value(bool) = .init(false),
-    thread: std.Thread = undefined,
+    /// The SPIR-V revision `pipeline` was made from (shader_build.Pass).
+    revision: u32 = 0,
 };
 
 passes: [max]Pass = @splat(.{}),
 len: usize = 0,
-/// The chain generation `passes` follow; null before the first.
-generation: ?u32 = null,
+/// The chain key `passes` follow (sync); null before the first.
+key: ?u32 = null,
 bundled: std.EnumArray(Scene, ?*Pipeline) = .initFill(null),
-job: ?*Job = null,
-/// Said once: without glslc, files never compile.
-missing_said: bool = false,
 ping: [2]?*c.SDL_GPUTexture = .{ null, null },
 ping_w: u32 = 0,
 ping_h: u32 = 0,
@@ -136,20 +123,17 @@ bloom_up: ?*Pipeline = null,
 started_ns: ?u64 = null,
 last_ns: u64 = 0,
 
-/// Brings the passes to the chain the core holds, and takes in a finished
-/// compile. Called each loop step, off the frame path.
-pub fn sync(post: *Post, gpa: std.mem.Allocator, io: std.Io, device: *c.SDL_GPUDevice, format: c.SDL_GPUTextureFormat, core: *pardes.Pardes) void {
-    if (post.job) |job| {
-        if (!job.done.load(.acquire)) return;
-        job.thread.join();
-        post.job = null;
-        post.collect(gpa, device, format, core, job);
-    }
-    const chain = &core.settings.post;
-    if (post.generation == chain.generation) return;
-    post.generation = chain.generation;
+/// Brings the passes to the chain a shell was given (shader_build.zig: a
+/// local core's, or an attached session's over the wire), whose `key` moves
+/// whenever the chain or a file's SPIR-V does. A file whose SPIR-V has not
+/// moved keeps its pipeline; one the GPU refuses keeps its last, and is said
+/// to `core`, or logged where there is none. Each loop step, off the frame
+/// path.
+pub fn sync(post: *Post, gpa: std.mem.Allocator, device: *c.SDL_GPUDevice, format: c.SDL_GPUTextureFormat, chain: []const Built, key: u32, core: ?*pardes.Pardes) void {
+    if (post.key == key) return;
+    post.key = key;
     var next: [max]Pass = @splat(.{});
-    for (chain.list(), 0..) |entry, i| {
+    for (chain, 0..) |entry, i| {
         if (entry.scene) |scene| {
             if (post.bundled.get(scene) == null) post.bundled.set(scene, makePipeline(device, format, bundled_spv.get(scene), if (scene == .bloom) 2 else 1, 4) catch null);
             if (scene == .bloom and post.bloom_down == null) {
@@ -161,11 +145,22 @@ pub fn sync(post: *Post, gpa: std.mem.Allocator, io: std.Io, device: *c.SDL_GPUD
         }
         // A file already in the chain keeps its pipeline and its path.
         for (post.passes[0..post.len]) |*pass| {
-            if (pass.scene != null or !std.mem.eql(u8, pass.path, entry.path.get())) continue;
+            if (pass.scene != null or !std.mem.eql(u8, pass.path, entry.path)) continue;
             next[i] = pass.*;
             pass.* = .{};
             break;
-        } else next[i] = .{ .path = gpa.dupe(u8, entry.path.get()) catch &.{} };
+        } else next[i] = .{ .path = gpa.dupe(u8, entry.path) catch &.{} };
+        const pass = &next[i];
+        if (pass.revision == entry.revision or entry.spirv.len == 0) continue;
+        pass.revision = entry.revision;
+        const made = makePipeline(device, format, entry.spirv, 1, 4) catch {
+            var buf: [512]u8 = undefined;
+            const text = std.fmt.bufPrint(&buf, "Shader {s}: the GPU refused it", .{entry.path}) catch "Shader: the GPU refused it";
+            if (core) |p| p.setMessage(p.active, text) else std.log.err("{s}", .{text});
+            continue;
+        };
+        if (pass.pipeline) |old| c.SDL_ReleaseGPUGraphicsPipeline(device, old);
+        pass.pipeline = made;
     }
     for (post.passes[0..post.len]) |pass| {
         if (pass.scene == null) if (pass.pipeline) |pipeline| c.SDL_ReleaseGPUGraphicsPipeline(device, pipeline);
@@ -179,127 +174,6 @@ pub fn sync(post: *Post, gpa: std.mem.Allocator, io: std.Io, device: *c.SDL_GPUD
         post.last_ns = 0;
         post.uniforms.frame = 0;
     }
-    // The new files compile together, on a thread of their own.
-    const job = gpa.create(Job) catch return;
-    job.* = .{ .io = io, .gpa = gpa };
-    for (post.passes[0..post.len]) |pass| {
-        if (pass.scene != null or pass.pipeline != null or pass.path.len == 0) continue;
-        job.paths[job.len] = gpa.dupe(u8, pass.path) catch continue;
-        job.len += 1;
-    }
-    if (job.len == 0) return gpa.destroy(job);
-    job.thread = std.Thread.spawn(.{}, compile, .{job}) catch {
-        for (job.paths[0..job.len]) |path| gpa.free(path);
-        return gpa.destroy(job);
-    };
-    post.job = job;
-}
-
-fn collect(post: *Post, gpa: std.mem.Allocator, device: *c.SDL_GPUDevice, format: c.SDL_GPUTextureFormat, core: *pardes.Pardes, job: *Job) void {
-    defer gpa.destroy(job);
-    for (job.paths[0..job.len], job.results[0..job.len]) |path, result| {
-        defer gpa.free(path);
-        const pass = for (post.passes[0..post.len]) |*pass| {
-            if (pass.scene == null and std.mem.eql(u8, pass.path, path)) break pass;
-        } else null;
-        var buf: [512]u8 = undefined;
-        switch (result) {
-            .none => {},
-            .missing => if (!post.missing_said) {
-                post.missing_said = true;
-                core.setMessage(core.active, "Shader: glslc is not on PATH, so Shadertoy files do not run (Crt does)");
-            },
-            .failed => |text| {
-                defer gpa.free(text);
-                // glslc's first line says where: in the file's own lines.
-                const first = std.mem.sliceTo(text, '\n');
-                core.setMessage(core.active, std.fmt.bufPrint(&buf, "Shader {s}: {s}", .{ path, first[0..@min(first.len, 400)] }) catch "Shader: compile failed");
-            },
-            .spirv => |spirv| {
-                defer gpa.free(spirv);
-                const target = pass orelse continue;
-                const made = makePipeline(device, format, spirv, 1, 4) catch {
-                    core.setMessage(core.active, std.fmt.bufPrint(&buf, "Shader {s}: the GPU refused it", .{path}) catch "Shader: the GPU refused it");
-                    continue;
-                };
-                if (target.pipeline) |old| c.SDL_ReleaseGPUGraphicsPipeline(device, old);
-                target.pipeline = made;
-            },
-        }
-    }
-}
-
-/// The compile thread: each file behind the prefix, through glslc with the
-/// build's own flags (build.zig compileGlsl), source on stdin, SPIR-V on
-/// stdout.
-fn compile(job: *Job) void {
-    defer job.done.store(true, .release);
-    defer {
-        var event = std.mem.zeroes(c.SDL_Event);
-        event.type = c.SDL_EVENT_USER;
-        _ = c.SDL_PushEvent(&event);
-    }
-    for (job.paths[0..job.len], 0..) |path, i| job.results[i] = compileOne(job.gpa, job.io, path);
-}
-
-fn compileOne(gpa: std.mem.Allocator, io: std.Io, path: []const u8) Job.Result {
-    var home_buf: [4096]u8 = undefined;
-    const expanded = if (std.mem.startsWith(u8, path, "~/")) if (libc.getenv("HOME")) |home|
-        std.fmt.bufPrint(&home_buf, "{s}/{s}", .{ std.mem.span(home), path[2..] }) catch path
-    else
-        path else path;
-    const body = filesystem.readFile(gpa, expanded) catch |err|
-        return .{ .failed = std.fmt.allocPrint(gpa, "cannot read it ({t})", .{err}) catch return .none };
-    defer gpa.free(body);
-    // The prefix and the file go to glslc on its stdin: no file of ours is
-    // written anywhere. std's spawn does every allocation before fork and
-    // searches PATH on the stack after it (Io/Threaded.zig spawnPosix,
-    // posixExecv), so a child forked from this thread cannot wedge on a
-    // lock another thread held (the forkShell hazard).
-    var child = std.process.spawn(io, .{
-        .argv = &.{ "glslc", "-fshader-stage=fragment", "-o", "-", "-" },
-        .stdin = .pipe,
-        .stdout = .pipe,
-        .stderr = .pipe,
-    }) catch |err| return switch (err) {
-        error.FileNotFound => .missing,
-        else => .{ .failed = std.fmt.allocPrint(gpa, "glslc did not run ({t})", .{err}) catch return .none },
-    };
-    defer child.kill(io);
-    // glslc reads all of its input before it writes a byte, so writing it
-    // first cannot deadlock against a full output pipe.
-    {
-        var stdin = child.stdin.?;
-        child.stdin = null;
-        defer stdin.close(io);
-        var buf: [4096]u8 = undefined;
-        var writer = stdin.writer(io, &buf);
-        writer.interface.writeAll(prefix) catch {};
-        writer.interface.writeAll(body) catch {};
-        writer.interface.flush() catch {};
-    }
-    var streams: std.Io.File.MultiReader.Buffer(2) = undefined;
-    var reader: std.Io.File.MultiReader = undefined;
-    reader.init(gpa, io, streams.toStreams(), &.{ child.stdout.?, child.stderr.? });
-    defer reader.deinit();
-    while (reader.fill(64, .none)) |_| {
-        if (reader.reader(0).buffered().len > 16 << 20 or reader.reader(1).buffered().len > 64 << 10)
-            return .{ .failed = gpa.dupe(u8, "glslc said too much") catch return .none };
-    } else |err| switch (err) {
-        error.EndOfStream => {},
-        else => return .{ .failed = gpa.dupe(u8, "reading glslc failed") catch return .none },
-    }
-    const term = child.wait(io) catch return .{ .failed = gpa.dupe(u8, "glslc did not finish") catch return .none };
-    const clean = switch (term) {
-        .exited => |code| code == 0,
-        else => false,
-    };
-    if (clean) return .{ .spirv = reader.toOwnedSlice(0) catch return .none };
-    const said = reader.toOwnedSlice(1) catch return .none;
-    // glslc calls its input <stdin>; the file is what a person knows.
-    defer gpa.free(said);
-    const named = std.mem.replaceOwned(u8, gpa, said, "<stdin>", path) catch return .{ .failed = gpa.dupe(u8, "glslc failed") catch return .none };
-    return .{ .failed = named };
 }
 
 fn makePipeline(device: *c.SDL_GPUDevice, format: c.SDL_GPUTextureFormat, frag: []const u8, samplers: u32, uniform_buffers: u32) !*Pipeline {
@@ -686,17 +560,6 @@ fn bloomStep(cmd: *c.SDL_GPUCommandBuffer, pipeline: *Pipeline, sampler: *c.SDL_
 pub fn deinit(post: *Post, gpa: std.mem.Allocator, device: *c.SDL_GPUDevice) void {
     for (post.bloom_tex) |texture| if (texture) |t| c.SDL_ReleaseGPUTexture(device, t);
     for ([_]?*Pipeline{ post.bloom_down, post.bloom_up }) |pipeline| if (pipeline) |p| c.SDL_ReleaseGPUGraphicsPipeline(device, p);
-    if (post.job) |job| {
-        job.thread.join();
-        for (job.paths[0..job.len], job.results[0..job.len]) |path, result| {
-            gpa.free(path);
-            switch (result) {
-                .spirv, .failed => |bytes| gpa.free(bytes),
-                .none, .missing => {},
-            }
-        }
-        gpa.destroy(job);
-    }
     for (post.passes[0..post.len]) |pass| {
         if (pass.scene == null) if (pass.pipeline) |pipeline| c.SDL_ReleaseGPUGraphicsPipeline(device, pipeline);
         gpa.free(pass.path);
@@ -801,36 +664,3 @@ test "a moved cursor becomes the previous one, stamped with the time of the move
     try std.testing.expectEqual([4]f32{ 1.0 / 255.0, 2.0 / 255.0, 3.0 / 255.0, 1 }, post.uniforms.selection_foreground_color);
 }
 
-test "ghostty's test shaders compile behind the prefix, and its invalid one fails with glslc's words" {
-    const gpa = std.testing.allocator;
-    const io = std.testing.io;
-    // ghostty's own test shaders, from its package in zig-pkg.
-    var pkg = std.Io.Dir.cwd().openDir(io, "zig-pkg", .{ .iterate = true }) catch return error.SkipZigTest;
-    defer pkg.close(io);
-    var it = pkg.iterate();
-    const ghostty = while (it.next(io) catch null) |entry| {
-        if (std.mem.startsWith(u8, entry.name, "ghostty-")) break entry.name;
-    } else return error.SkipZigTest;
-    var buf: [512]u8 = undefined;
-    for ([_][]const u8{ "crt", "focus", "invalid" }) |name| {
-        const path = try std.fmt.bufPrint(&buf, "zig-pkg/{s}/src/renderer/shaders/test_shadertoy_{s}.glsl", .{ ghostty, name });
-        const result = compileOne(gpa, io, path);
-        switch (result) {
-            .missing => return error.SkipZigTest, // no glslc here
-            .spirv => |spirv| {
-                defer gpa.free(spirv);
-                try std.testing.expect(!std.mem.eql(u8, name, "invalid"));
-                // SPIR-V's magic number, little-endian.
-                try std.testing.expectEqual(@as(u32, 0x07230203), std.mem.readInt(u32, spirv[0..4], .little));
-            },
-            .failed => |text| {
-                defer gpa.free(text);
-                try std.testing.expectEqualStrings("invalid", name);
-                // Named by the file, not by glslc's <stdin>.
-                try std.testing.expect(std.mem.indexOf(u8, text, path) != null);
-                try std.testing.expect(std.mem.indexOf(u8, text, "<stdin>") == null);
-            },
-            .none => return error.TestUnexpectedResult,
-        }
-    }
-}
diff --git a/src/gui/gui.zig b/src/gui/gui.zig
index 127f9b5d..2d31416b 100644
--- a/src/gui/gui.zig
+++ b/src/gui/gui.zig
@@ -11,6 +11,7 @@ const config = @import("../config.zig");
 const look = @import("../look.zig");
 const message = pardes.Messages.Message;
 const file_watch = @import("../file_watch.zig");
+const shader_build = @import("../shader_build.zig");
 const deck = @import("deck.zig");
 const Post = @import("Post.zig");
 const pet = @import("pet.zig");
@@ -2693,6 +2694,7 @@ fn localSession(
         .test_hz = if (std.c.getenv("PARDES_TEST_HZ")) |hz| @max(1, std.fmt.parseInt(u64, std.mem.span(hz), 10) catch 60) else 60,
     };
     defer shell.shutdownPtys();
+    defer shell.shaders.deinit(gpa, inotify_fd, &watches);
     // A virtual clock never blinks: goldens and feel series hold still.
     g.no_blink = shell.test_clock != null;
     const host = shell.host();
@@ -2862,6 +2864,7 @@ fn attachedLoop(gpa: std.mem.Allocator, g: *Gui, client: *detached_client.Client
                     in.post(.{ .paste = text });
                 },
                 .open_link => |url| look.openLink(url),
+                .post => dirty = true,
             }
         }
         try link;
@@ -2877,7 +2880,15 @@ fn attachedLoop(gpa: std.mem.Allocator, g: *Gui, client: *detached_client.Client
             geom = now;
             try attachedResize(client, g, now.cols, now.rows);
         }
-        if (!dirty) continue;
+        // The session's chain, its files compiled there.
+        g.post.sync(gpa, g.device, g.swapchain_format, client.post.list(), client.post_key, null);
+        if (!dirty) {
+            // As a local window does: a chain that moves on its own redraws
+            // alone, once a refresh.
+            if (g.post.animating(client.post.animation) and g.scene_tex != null and
+                c.SDL_GetTicksNS() -| g.post.last_ns + std.time.ns_per_ms >= refreshNs(g)) redrawPost(g, gpa);
+            continue;
+        }
         dirty = false;
         paintAttached(g, gpa, client);
     }
@@ -3993,6 +4004,8 @@ const Shell = struct {
     pipe_tasks: *PipeTasks,
     inotify_fd: c_int,
     watches: *file_watch.Table,
+    /// The post chain's files as SPIR-V.
+    shaders: shader_build = .{},
     threads_ok: bool = false,
     test_mode: bool = false,
     feed: StdinFeed = .{},
@@ -4209,11 +4222,19 @@ const Shell = struct {
             .files_changed => check_files = true,
             .fs_ready => {},
         };
+        if (check_files) s.shaders.recheck();
         if (check_files and file_watch.reloadChanged(s.core, s.io, s.gpa, s.watches))
             s.queue.push(.files_changed);
     }
 };
 
+/// A post chain compile is done (shader_build.zig): the loop's SDL wait ends.
+fn wakeSdl(_: ?*anyopaque) void {
+    var event = std.mem.zeroes(c.SDL_Event);
+    event.type = c.SDL_EVENT_USER;
+    _ = c.SDL_PushEvent(&event);
+}
+
 fn wakeFs(ctx: ?*anyopaque) void {
     const q: *Queue = @ptrCast(@alignCast(ctx.?));
     q.push(.fs_ready);
@@ -4356,7 +4377,11 @@ fn pollFrame(ctx: ?*anyopaque) void {
     const geom = windowCells(g);
     if (updateCoreResize(core, geom.cols, geom.rows, g.cell_w, g.cell_h, g.tagline_width, g.tagline_height)) resetScroll(g);
     stepScroll(g, core, s.gpa);
-    g.post.sync(s.gpa, s.io, g.device, g.swapchain_format, core);
+    _ = s.shaders.sync(s.gpa, s.io, core, s.inotify_fd, s.watches, .{ .call = wakeSdl });
+    {
+        var passes: [Post.max]shader_build.Pass = undefined;
+        g.post.sync(s.gpa, g.device, g.swapchain_format, s.shaders.view(&core.settings.post, &passes), s.shaders.revision, core);
+    }
     if (s.test_mode) writeTestStatus(g, core, s.presented);
     // Between the core's frames, a chain that moves on its own redraws
     // alone, once a refresh of the display the window is on (a millisecond
@@ -5805,8 +5830,9 @@ fn renderFrame(
     const win_w: f32 = @floatFromInt(sw);
     const win_h: f32 = @floatFromInt(sh);
 
-    // The post chain runs over a local frame, from the scene texture.
-    var scene_on = core != null and g.post.ready();
+    // The post chain runs from the scene texture, over a local frame or an
+    // attached session's alike.
+    var scene_on = g.post.ready();
     if (scene_on) ensureSceneTexture(g, sw, sh) catch {
         g.scene_target_failed = true;
         scene_on = false;
@@ -6179,19 +6205,17 @@ fn renderFrame(
     c.SDL_EndGPURenderPass(rp);
 
     if (scene_on) {
-        if (core) |p| {
-            var boxes: [max_cursors]CursorBox = undefined;
-            const cursors = if (cursors_shown) cursorBoxes(g, surface, layout, win_h, &boxes) else 0;
-            var spare: [Post.max_spare][4]f32 = undefined;
-            g.post.describe(.{
-                .cursor = if (cursors > 0) boxes[0] else null,
-                .spare = spare[0..postSpare(surface, layout, &spare)],
-                .background = chrome.page orelse bg_default,
-                .foreground = chrome.fg orelse fg_default,
-                .selection_foreground = p.theme().sel_fg,
-                .selection_background = p.theme().sel_bg,
-            });
-        }
+        var boxes: [max_cursors]CursorBox = undefined;
+        const cursors = if (cursors_shown) cursorBoxes(g, surface, layout, win_h, &boxes) else 0;
+        var spare: [Post.max_spare][4]f32 = undefined;
+        g.post.describe(.{
+            .cursor = if (cursors > 0) boxes[0] else null,
+            .spare = spare[0..postSpare(surface, layout, &spare)],
+            .background = chrome.page orelse bg_default,
+            .foreground = chrome.fg orelse fg_default,
+            .selection_foreground = if (core) |p| p.theme().sel_fg else chrome.sel_fg,
+            .selection_background = if (core) |p| p.theme().sel_bg else chrome.sel_bg,
+        });
         const chain_zone = tracy.zone(@src(), "post chain");
         defer chain_zone.end();
         g.post.scale = display_scale;