From 826ec1fac3c012e71ab34207328faaac5b5827b7 Mon Sep 17 00:00:00 2001 From: Gabriel Schneider Date: Tue, 29 Sep 2026 09:43:05 -0300 Subject: Shader files compile again on save; an attached GUI runs the session's post chain shader_build.zig takes the Shadertoy compile out of the GUI's Post: the process that holds the core compiles each chain file behind the prefix on a thread, watches its directory (file_watch's new shader slots), and on a change reads and hashes each file, compiling only one whose bytes moved. A failed compile keeps the last good SPIR-V and is said once; the same bytes are never compiled or reported twice. A file is read with std, not fs.zig's readFile, whose turn hand-off a worker thread does not hold. A detached session compiles for its frontends and sends them the chain (wire post: each pass's scene and level or the file's SPIR-V, and ShaderAnimation) on attach and on every change, so an attached GUI runs the same passes, levels and animation as a local one while still reading no disk and running no program. The attached GUI describes its frames to the chain from the session's chrome and redraws an animating chain on its own. --- src/shader_build.zig | 402 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 402 insertions(+) create mode 100644 src/shader_build.zig (limited to 'src/shader_build.zig') diff --git a/src/shader_build.zig b/src/shader_build.zig new file mode 100644 index 00000000..8aa7eb5e --- /dev/null +++ b/src/shader_build.zig @@ -0,0 +1,402 @@ +//! The post chain's Shadertoy files as SPIR-V (docs/render-pipeline.md ยง6), +//! for whichever process holds the core: a local GUI makes its pipelines +//! from them, a detached session sends them to the GUIs attached to it +//! (wire `post`), which never read a disk or run a program themselves. +//! +//! Each file compiles behind the prefix by the build's own glslc invocation, +//! on a thread of its own and never on the frame path, when it joins the +//! chain and again whenever a save changes its bytes: the files' directories +//! are watched (file_watch.watchShaders) and a change there has every file +//! read and hashed, an unchanged one compiling nothing. A compile that fails +//! keeps the last good SPIR-V and is said once, and glslc missing is said +//! once for good. +const std = @import("std"); +const libc = std.c; +const pardes = @import("pardes.zig"); +const file_watch = @import("file_watch.zig"); + +const Build = @This(); +const Chain = pardes.config.Runtime.Post; +const Scene = pardes.config.Runtime.Scene; +pub const max = Chain.max; + +const prefix = @embedFile("post-prefix.glsl"); + +/// A pass as a shell draws it: a bundled scene at its level, or a file with +/// its last good SPIR-V (empty until a compile succeeds), whose `revision` +/// moves each time that SPIR-V does. Slices borrow the build's (or, in an +/// attached GUI, the client's) storage until its next change. +pub const Pass = struct { + scene: ?Scene = null, + level: u8 = 0, + path: []const u8 = &.{}, + spirv: []const u8 = &.{}, + revision: u32 = 0, +}; + +/// Who to wake when a compile is done: the host's loop. +pub const Wake = struct { + ctx: ?*anyopaque = null, + call: ?*const fn (?*anyopaque) void = null, +}; + +const File = struct { + path: []u8 = &.{}, + /// The hash of the source last compiled, good or not: the same bytes + /// saved again compile nothing and say nothing twice. + source: ?u64 = null, + spirv: []u8 = &.{}, + revision: u32 = 0, + /// Wants a compile: new to the chain, or its directory changed. + stale: bool = true, +}; + +const Result = union(enum) { + none, + unchanged, + missing, + failed: struct { text: []u8, source: u64 }, + spirv: struct { bytes: []u8, source: u64 }, +}; + +/// One compile of the stale files, on its own thread. +const Job = struct { + io: std.Io, + gpa: std.mem.Allocator, + wake: Wake, + paths: [max][]u8 = @splat(&.{}), + known: [max]?u64 = @splat(null), + results: [max]Result = @splat(.none), + len: usize = 0, + done: std.atomic.Value(bool) = .init(false), + thread: std.Thread = undefined, +}; + +files: [max]File = @splat(.{}), +len: usize = 0, +/// The chain generation `files` follow; null before the first. +generation: ?u32 = null, +/// Moves whenever what `view` gives does: the chain or a file's SPIR-V. +revision: u32 = 0, +job: ?*Job = null, +missing_said: bool = false, + +/// Brings the files to the chain the core holds, watches their directories, +/// takes in a finished compile and starts the next. Each loop step, off the +/// frame path. True when a compile was taken in: what it said wants a frame. +pub fn sync(b: *Build, gpa: std.mem.Allocator, io: std.Io, core: *pardes.Pardes, fd: c_int, watches: *file_watch.Table, wake: Wake) bool { + var collected = false; + if (b.job) |job| { + if (!job.done.load(.acquire)) return false; + job.thread.join(); + b.job = null; + b.collect(gpa, core, job); + collected = true; + } + const chain = &core.settings.post; + if (b.generation != chain.generation) { + b.generation = chain.generation; + b.revision +%= 1; + var next: [max]File = @splat(.{}); + var len: usize = 0; + var paths: [max][]const u8 = undefined; + for (chain.list()) |entry| { + if (entry.scene != null) continue; + next[len] = for (b.files[0..b.len]) |*file| { + if (!std.mem.eql(u8, file.path, entry.path.get())) continue; + const kept = file.*; + file.* = .{}; + break kept; + } else .{ .path = gpa.dupe(u8, entry.path.get()) catch continue }; + paths[len] = next[len].path; + len += 1; + } + for (b.files[0..b.len]) |file| free(gpa, file); + b.files = next; + b.len = len; + var bufs: [max][4096]u8 = undefined; + var expanded: [max][]const u8 = undefined; + for (paths[0..len], 0..) |path, i| expanded[i] = expand(&bufs[i], path); + file_watch.watchShaders(fd, watches, expanded[0..len]); + } + var stale = false; + for (b.files[0..b.len]) |file| stale = stale or file.stale; + if (!stale) return collected; + const job = gpa.create(Job) catch return collected; + job.* = .{ .io = io, .gpa = gpa, .wake = wake }; + for (b.files[0..b.len]) |*file| { + if (!file.stale) continue; + job.paths[job.len] = gpa.dupe(u8, file.path) catch continue; + job.known[job.len] = file.source; + job.len += 1; + file.stale = false; + } + job.thread = std.Thread.spawn(.{}, compile, .{job}) catch { + for (job.paths[0..job.len]) |path| gpa.free(path); + gpa.destroy(job); + return collected; + }; + b.job = job; + return collected; +} + +/// A watched directory changed: every file is read again at the next sync. +pub fn recheck(b: *Build) void { + for (b.files[0..b.len]) |*file| file.stale = true; +} + +/// The chain as passes, in its order. +pub fn view(b: *const Build, chain: *const Chain, out: *[max]Pass) []const Pass { + for (chain.list(), 0..) |entry, i| { + out[i] = if (entry.scene) |scene| .{ .scene = scene, .level = entry.level } else for (b.files[0..b.len]) |file| { + if (std.mem.eql(u8, file.path, entry.path.get())) break .{ .path = file.path, .spirv = file.spirv, .revision = file.revision }; + } else .{ .path = entry.path.get() }; + } + return out[0..chain.len]; +} + +pub fn deinit(b: *Build, gpa: std.mem.Allocator, fd: c_int, watches: *file_watch.Table) void { + if (b.job) |job| { + job.thread.join(); + freeJob(gpa, job); + } + for (b.files[0..b.len]) |file| free(gpa, file); + b.len = 0; + file_watch.watchShaders(fd, watches, &.{}); +} + +fn free(gpa: std.mem.Allocator, file: File) void { + gpa.free(file.path); + gpa.free(file.spirv); +} + +fn freeJob(gpa: std.mem.Allocator, job: *Job) void { + for (job.paths[0..job.len], job.results[0..job.len]) |path, result| { + gpa.free(path); + switch (result) { + .spirv => |r| gpa.free(r.bytes), + .failed => |r| gpa.free(r.text), + .none, .unchanged, .missing => {}, + } + } + gpa.destroy(job); +} + +fn collect(b: *Build, gpa: std.mem.Allocator, core: *pardes.Pardes, job: *Job) void { + defer gpa.destroy(job); + for (job.paths[0..job.len], job.results[0..job.len]) |path, result| { + defer gpa.free(path); + const file = for (b.files[0..b.len]) |*file| { + if (std.mem.eql(u8, file.path, path)) break file; + } else null; + var buf: [512]u8 = undefined; + switch (result) { + .none, .unchanged => {}, + .missing => if (!b.missing_said) { + b.missing_said = true; + core.setMessage(core.active, "Shader: glslc is not on PATH, so Shadertoy files do not run (Crt does)"); + }, + .failed => |r| { + defer gpa.free(r.text); + if (file) |f| f.source = r.source; + // glslc's first line says where: in the file's own lines. + const first = std.mem.sliceTo(r.text, '\n'); + core.setMessage(core.active, std.fmt.bufPrint(&buf, "Shader {s}: {s}", .{ path, first[0..@min(first.len, 400)] }) catch "Shader: compile failed"); + }, + .spirv => |r| { + const f = file orelse { + gpa.free(r.bytes); + continue; + }; + gpa.free(f.spirv); + f.spirv = r.bytes; + f.source = r.source; + f.revision +%= 1; + b.revision +%= 1; + }, + } + } +} + +/// `~/` is the home directory. +fn expand(buf: []u8, path: []const u8) []const u8 { + if (!std.mem.startsWith(u8, path, "~/")) return path; + const home = libc.getenv("HOME") orelse return path; + return std.fmt.bufPrint(buf, "{s}/{s}", .{ std.mem.span(home), path[2..] }) catch path; +} + +/// The compile thread: each file behind the prefix, through glslc with the +/// build's own flags (build.zig compileGlsl), source on stdin, SPIR-V on +/// stdout. +fn compile(job: *Job) void { + for (job.paths[0..job.len], 0..) |path, i| job.results[i] = compileOne(job.gpa, job.io, path, job.known[i]); + job.done.store(true, .release); + if (job.wake.call) |call| call(job.wake.ctx); +} + +fn compileOne(gpa: std.mem.Allocator, io: std.Io, path: []const u8, known: ?u64) Result { + var home_buf: [4096]u8 = undefined; + // Not fs.zig's readFile: that hands the core's turn over for the read, + // a turn this thread never holds. + const body = std.Io.Dir.cwd().readFileAlloc(io, expand(&home_buf, path), gpa, .limited(1 << 20)) catch |err| + return .{ .failed = .{ .text = std.fmt.allocPrint(gpa, "cannot read it ({t})", .{err}) catch return .none, .source = 0 } }; + defer gpa.free(body); + const source = std.hash.Wyhash.hash(0, body); + if (known == source) return .unchanged; + // The prefix and the file go to glslc on its stdin: no file of ours is + // written anywhere. std's spawn does every allocation before fork and + // searches PATH on the stack after it (Io/Threaded.zig spawnPosix, + // posixExecv), so a child forked from this thread cannot wedge on a + // lock another thread held (the forkShell hazard). + var child = std.process.spawn(io, .{ + .argv = &.{ "glslc", "-fshader-stage=fragment", "-o", "-", "-" }, + .stdin = .pipe, + .stdout = .pipe, + .stderr = .pipe, + }) catch |err| return switch (err) { + error.FileNotFound => .missing, + else => .{ .failed = .{ .text = std.fmt.allocPrint(gpa, "glslc did not run ({t})", .{err}) catch return .none, .source = source } }, + }; + defer child.kill(io); + // glslc reads all of its input before it writes a byte, so writing it + // first cannot deadlock against a full output pipe. + { + var stdin = child.stdin.?; + child.stdin = null; + defer stdin.close(io); + var buf: [4096]u8 = undefined; + var writer = stdin.writer(io, &buf); + writer.interface.writeAll(prefix) catch {}; + writer.interface.writeAll(body) catch {}; + writer.interface.flush() catch {}; + } + var streams: std.Io.File.MultiReader.Buffer(2) = undefined; + var reader: std.Io.File.MultiReader = undefined; + reader.init(gpa, io, streams.toStreams(), &.{ child.stdout.?, child.stderr.? }); + defer reader.deinit(); + const failed = struct { + fn of(a: std.mem.Allocator, text: []const u8, hash: u64) Result { + return .{ .failed = .{ .text = a.dupe(u8, text) catch return .none, .source = hash } }; + } + }.of; + while (reader.fill(64, .none)) |_| { + if (reader.reader(0).buffered().len > 16 << 20 or reader.reader(1).buffered().len > 64 << 10) + return failed(gpa, "glslc said too much", source); + } else |err| switch (err) { + error.EndOfStream => {}, + else => return failed(gpa, "reading glslc failed", source), + } + const term = child.wait(io) catch return failed(gpa, "glslc did not finish", source); + const clean = switch (term) { + .exited => |code| code == 0, + else => false, + }; + if (clean) return .{ .spirv = .{ .bytes = reader.toOwnedSlice(0) catch return .none, .source = source } }; + const said = reader.toOwnedSlice(1) catch return .none; + // glslc calls its input ; the file is what a person knows. + defer gpa.free(said); + const named = std.mem.replaceOwned(u8, gpa, said, "", path) catch return failed(gpa, "glslc failed", source); + return .{ .failed = .{ .text = named, .source = source } }; +} + +test "ghostty's test shaders compile behind the prefix, and its invalid one fails with glslc's words" { + const gpa = std.testing.allocator; + const io = std.testing.io; + // ghostty's own test shaders, from its package in zig-pkg. + var pkg = std.Io.Dir.cwd().openDir(io, "zig-pkg", .{ .iterate = true }) catch return error.SkipZigTest; + defer pkg.close(io); + var it = pkg.iterate(); + const ghostty = while (it.next(io) catch null) |entry| { + if (std.mem.startsWith(u8, entry.name, "ghostty-")) break entry.name; + } else return error.SkipZigTest; + var buf: [512]u8 = undefined; + for ([_][]const u8{ "crt", "focus", "invalid" }) |name| { + const path = try std.fmt.bufPrint(&buf, "zig-pkg/{s}/src/renderer/shaders/test_shadertoy_{s}.glsl", .{ ghostty, name }); + switch (compileOne(gpa, io, path, null)) { + .missing => return error.SkipZigTest, // no glslc here + .spirv => |r| { + defer gpa.free(r.bytes); + try std.testing.expect(!std.mem.eql(u8, name, "invalid")); + // SPIR-V's magic number, little-endian. + try std.testing.expectEqual(@as(u32, 0x07230203), std.mem.readInt(u32, r.bytes[0..4], .little)); + // The same bytes again compile nothing. + try std.testing.expectEqual(Result.unchanged, compileOne(gpa, io, path, r.source)); + }, + .failed => |r| { + defer gpa.free(r.text); + try std.testing.expectEqualStrings("invalid", name); + // Named by the file, not by glslc's . + try std.testing.expect(std.mem.indexOf(u8, r.text, path) != null); + try std.testing.expect(std.mem.indexOf(u8, r.text, "") == null); + }, + .none, .unchanged => return error.TestUnexpectedResult, + } + } +} + +test "a save that changes a file compiles it again, a failed one keeps the last good SPIR-V and is said once" { + const gpa = std.testing.allocator; + const io = std.testing.io; + const core = try pardes.Pardes.init(gpa, .{ .cols = 80, .rows = 24 }); + defer core.deinit(); + _ = try core.newShell(1, ""); + const pane = core.panes[core.active].?; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var dir_buf: [4096]u8 = undefined; + const dir = dir_buf[0..try tmp.dir.realPath(io, &dir_buf)]; + var path_buf: [4200]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/wave.glsl", .{dir}); + const good = "void mainImage(out vec4 o, in vec2 p) { o = texture(iChannel0, p / iResolution.xy); }\n"; + try tmp.dir.writeFile(io, .{ .sub_path = "wave.glsl", .data = good }); + try std.testing.expect(core.settings.post.set(null, path, true)); + + const fd = file_watch.init(true); + defer if (fd >= 0) { + _ = libc.close(fd); + }; + var watches: file_watch.Table = @splat(null); + var b: Build = .{}; + defer b.deinit(gpa, fd, &watches); + const settle = struct { + fn run(build: *Build, c: *pardes.Pardes, f: c_int, w: *file_watch.Table) void { + _ = build.sync(std.testing.allocator, std.testing.io, c, f, w, .{}); + while (build.job) |job| { + while (!job.done.load(.acquire)) std.Thread.yield() catch {}; + _ = build.sync(std.testing.allocator, std.testing.io, c, f, w, .{}); + } + } + }.run; + settle(&b, core, fd, &watches); + if (b.missing_said) return error.SkipZigTest; // no glslc here + var out: [max]Pass = undefined; + const first = b.view(&core.settings.post, &out)[0]; + try std.testing.expect(first.spirv.len != 0); + if (file_watch.supported) try std.testing.expect(watches[file_watch.shader_slot] != null); + + // Unchanged bytes: nothing moves. + const at = b.revision; + b.recheck(); + settle(&b, core, fd, &watches); + try std.testing.expectEqual(at, b.revision); + + // A bad save: the last good SPIR-V stays, the error is said, and the + // same bad bytes again say nothing more. + try tmp.dir.writeFile(io, .{ .sub_path = "wave.glsl", .data = "void mainImage(" }); + pane.msg_len = 0; + b.recheck(); + settle(&b, core, fd, &watches); + try std.testing.expectEqual(at, b.revision); + try std.testing.expect(std.mem.startsWith(u8, pane.msg[0..pane.msg_len], "Shader ")); + pane.msg_len = 0; + b.recheck(); + settle(&b, core, fd, &watches); + try std.testing.expectEqual(@as(u16, 0), pane.msg_len); + + // A good save again: new SPIR-V, a new revision. + try tmp.dir.writeFile(io, .{ .sub_path = "wave.glsl", .data = "// saved again\n" ++ good }); + b.recheck(); + settle(&b, core, fd, &watches); + const second = b.view(&core.settings.post, &out)[0]; + try std.testing.expect(second.revision != first.revision and b.revision != at); +} -- cgit v1.3