summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorGabriel Schneider <[email protected]>2026-11-17 01:01:34 -0300
committerGabriel Schneider <[email protected]>2026-11-17 12:15:01 -0300
commit562b503d5007e3afe1de426fc1175eadfbbb45c4 (patch)
tree8f36dbdec4f0e8602902a656ddf036ee1e8d29b9 /src
parentf2829440b0f933422e1da5394870505d3c2d67d6 (diff)
downloadsketchbook-562b503d5007e3afe1de426fc1175eadfbbb45c4.tar.gz
sketchbook-562b503d5007e3afe1de426fc1175eadfbbb45c4.zip
audio: a GPU buffer streamed within the frame, at a lead the host measures
A compute shader's output was heard only after the sketch read it back (f.download, a frame later) and pushed it behind a lead of its own choosing; spheres kept 100 ms, and a change it made left for the device 96 ms later on average (measured, dummy driver, 60 Hz). f.audioBuffer(key, buffer, .{ .count, .offset }) streams `count` f32 a shader wrote: the host copies them back while it runs the frame's commands, waiting on the copy's fence (Executor.read: 0.2 ms on average here, 2 ms at the 99th percentile on a loaded machine), and queues them on the stream, so nothing is in flight between frames and the sketch keeps no books. Input.lead says how far ahead to make sound: the slowest of the last 64 frames' worth of samples (timed between the ends of the frames' commands, where the pushes are), plus the device's pull (what SDL takes from a stream at once, read from the device: 1024 here), plus half a pull for each underrun, a sample less each frame after; capped at 8192. A sketch pushes `in.lead -| in.stream(key).queued` a frame, and its change leaves after about what is queued ahead of it, a pull: 19 to 27 ms on average with SDL's 1024-sample pulls, 6.3 to 6.5 ms with 256 (SDL_AUDIO_DEVICE_SAMPLE_FRAMES; 17 under heavy load). Stream.underruns counts the pushes that found a stream empty though the frame before had fed it (a gap). The voices keep their 2048 samples: mixed to the lead, a one-shot longer than it is mixed over several frames, and voices.script's expect-audio and expect-no-audio, which look one frame at a time, failed. On the web, audio.buffer reads the buffer back with mapAsync and queues it on arrival, in order, counting it as queued meanwhile; in.lead stays at 100 ms there. The measurements (a probe patched into the host for the purpose, kept out of the tree) are in the 02-pardes-code scratch, .scratch/sketch/gpusynth. Tests: the lead's arithmetic (audio.zig, in test-audio's filter); spheres' play script checks its stream is fed. zig build test: 164/164 steps, 49/49 tests run (the rest cached); zig build test-audio: 48/48 steps, with the voices sketch's wasm, so web.zig compiles; voices.script passes four runs in four. web/host.js: node --check only, no browser run. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_016Y9MNBECMWcJD5pdS4SR6T
Diffstat (limited to 'src')
-rw-r--r--src/abi.zig14
-rw-r--r--src/gpu/Executor.zig26
-rw-r--r--src/host/Host.zig5
-rw-r--r--src/host/audio.zig89
-rw-r--r--src/sketch/root.zig7
-rw-r--r--src/sketch/web.zig6
6 files changed, 141 insertions, 6 deletions
diff --git a/src/abi.zig b/src/abi.zig
index 35c399b..271c6dc 100644
--- a/src/abi.zig
+++ b/src/abi.zig
@@ -165,6 +165,9 @@ pub const Stream = struct {
queued: u32 = 0,
/// Samples pushed since the stream was last cleared.
pushed: u64 = 0,
+ /// Pushes that found it empty though the frame before had pushed to
+ /// it: each a gap in its sound.
+ underruns: u32 = 0,
};
/// Everything a sketch learns about the world for one call.
@@ -218,6 +221,11 @@ pub const Input = struct {
keys: std.StaticBitSet(256) = .empty,
streams: []const Stream = &.{},
sample_rate: u32 = 48_000,
+ /// How many samples a stream should hold after this frame's push, as
+ /// the host measures it: enough to outlast the slowest recent frame and
+ /// the device's next pull, and more after an underrun. A sketch that
+ /// makes its sound as it goes pushes `lead -| stream(key).queued`.
+ lead: u32 = 4800,
/// Mono samples at `sample_rate` recorded since the previous frame, while
/// `draw` declares `Command.microphone`. Only `draw` sees them: they come
/// from a device, not from events, so they do not replay.
@@ -377,6 +385,12 @@ pub const Command = union(enum) {
voice: Voice,
/// Mono samples at `Input.sample_rate` for stream `key`.
audio: struct { key: u32, samples: []const f32, clear: bool = false },
+ /// `count` mono samples (f32) that a compute shader left in `buffer`
+ /// at `offset`, for stream `key`: queued like `audio`'s once the GPU
+ /// has made them. Natively that is within this frame (the host waits
+ /// for the GPU's work so far); on the web they arrive a frame or two
+ /// later and count as queued meanwhile.
+ audio_buffer: struct { key: u32, buffer: Buffer, offset: u32 = 0, count: u32 },
/// The gain of a group of voices (`Voice.group`) this frame; a group
/// not given one plays at 1. Declared like a voice, so it replays.
gain: struct { group: u3, gain: f32 },
diff --git a/src/gpu/Executor.zig b/src/gpu/Executor.zig
index 460b2d5..797dcbf 100644
--- a/src/gpu/Executor.zig
+++ b/src/gpu/Executor.zig
@@ -503,6 +503,32 @@ pub fn download(g: *Gpu, d: @FieldType(abi.Command, "download")) !void {
};
}
+/// Copies `out.len` bytes of a buffer from `offset` back now, waiting for
+/// the GPU to finish the work submitted before (sound that must be queued
+/// this frame).
+pub fn read(g: *Gpu, buffer: abi.Buffer, offset: u32, out: []u8) !void {
+ const source = g.storage(buffer) orelse return error.UnknownBuffer;
+ const size: u32 = @intCast(out.len);
+ if (size == 0 or @as(u64, offset) + size > source.size) return error.DownloadOutOfBounds;
+ const transfer = for (g.spare.items, 0..) |t, i| {
+ if (t.size >= size) break g.spare.swapRemove(i);
+ } else Transfer{
+ .buffer = c.SDL_CreateGPUTransferBuffer(g.device, &.{ .usage = c.SDL_GPU_TRANSFERBUFFERUSAGE_DOWNLOAD, .size = size, .props = 0 }) orelse return error.GpuBuffer,
+ .size = size,
+ };
+ defer g.spare.append(g.gpa, transfer) catch c.SDL_ReleaseGPUTransferBuffer(g.device, transfer.buffer);
+ const cmd = c.SDL_AcquireGPUCommandBuffer(g.device) orelse return error.GpuCommandBuffer;
+ const copy = c.SDL_BeginGPUCopyPass(cmd);
+ c.SDL_DownloadFromGPUBuffer(copy, &.{ .buffer = source.buffer, .offset = offset, .size = size }, &.{ .transfer_buffer = transfer.buffer, .offset = 0 });
+ c.SDL_EndGPUCopyPass(copy);
+ const fence = c.SDL_SubmitGPUCommandBufferAndAcquireFence(cmd) orelse return error.GpuSubmit;
+ defer c.SDL_ReleaseGPUFence(g.device, fence);
+ if (!c.SDL_WaitForGPUFences(g.device, true, &fence, 1)) return error.GpuFence;
+ const mapped: [*]const u8 = @ptrCast(c.SDL_MapGPUTransferBuffer(g.device, transfer.buffer, false) orelse return error.GpuMap);
+ defer c.SDL_UnmapGPUTransferBuffer(g.device, transfer.buffer);
+ @memcpy(out, mapped[0..size]);
+}
+
/// The downloads the GPU has finished, copied into `arena`, in the order
/// they were asked for.
pub fn collect(g: *Gpu, arena: std.mem.Allocator) ![]const abi.Download {
diff --git a/src/host/Host.zig b/src/host/Host.zig
index 4f1214a..dc21cc0 100644
--- a/src/host/Host.zig
+++ b/src/host/Host.zig
@@ -478,6 +478,7 @@ fn draw(self: *Host, now: u64, started: u64) !u64 {
}
}
self.input.streams = self.audio.streamStatus();
+ self.input.lead = self.audio.lead;
self.input.sample_rate = sound.sample_rate;
self.input.microphone = self.audio.record(arena);
self.input.downloads = if (self.passes) |*g| try g.collect(arena) else &.{};
@@ -505,6 +506,10 @@ fn draw(self: *Host, now: u64, started: u64) !u64 {
.voice => |voice| self.audio.declare(voice),
.gain => |g| self.audio.gains[g.group] = g.gain,
.audio => |a| self.audio.push(a.key, a.samples, a.clear),
+ .audio_buffer => |a| if (self.passes) |*g| if (a.count > 0) {
+ const samples = try arena.alloc(f32, a.count);
+ if (g.read(a.buffer, a.offset, std.mem.sliceAsBytes(samples))) self.audio.push(a.key, samples, false) else |err| self.print("gpu audio: {t} ({s})", .{ err, c.SDL_GetError() });
+ },
.quit => self.quit = true,
.gpu => |pass| if (self.passes) |*g| self.runGpu(g, pass) catch |err| self.print("gpu pass: {t} ({s})", .{ err, c.SDL_GetError() }),
.post_process => |p| if (self.passes) |*g| self.runPostProcess(g, p.source, p.target, p.effect) catch |err| self.print("gpu postprocess: {t} ({s})", .{ err, c.SDL_GetError() }),
diff --git a/src/host/audio.zig b/src/host/audio.zig
index bf5fdc4..e95e38e 100644
--- a/src/host/audio.zig
+++ b/src/host/audio.zig
@@ -28,6 +28,19 @@ pub const Audio = struct {
heard_gains: [8]f32 = @splat(1),
streams: std.ArrayList(Stream) = .empty,
status: std.ArrayList(abi.Stream) = .empty,
+ /// Samples the device takes from each stream at once.
+ pull: u32 = 1024,
+ /// The wall-clock time (ns) between the ends of the last frames'
+ /// commands, where pushes are done; and when the last ended.
+ intervals: [64]u64 = @splat(0),
+ frame: u64 = 0,
+ last_frame: u64 = 0,
+ /// Grown by underruns, shrinking back while there are none.
+ margin: u32 = 0,
+ /// A stream ran dry since the lead was last worked out.
+ dry: bool = false,
+ /// `Input.lead`: see `lead`.
+ lead: u32 = 4800,
scope: Scope = .{},
/// The default recording device, bound to this stream while declared.
microphone: ?*c.SDL_AudioStream = null,
@@ -44,12 +57,25 @@ pub const Audio = struct {
/// Mixed since a script last asked (`expect-audio`).
sounded: bool = false,
};
- const Stream = struct { key: u32, out: *c.SDL_AudioStream, pushed: u64 = 0 };
+ const Stream = struct {
+ key: u32,
+ out: *c.SDL_AudioStream,
+ pushed: u64 = 0,
+ /// Pushed to this frame, and the last.
+ fed: bool = false,
+ was_fed: bool = false,
+ underruns: u32 = 0,
+ };
pub fn open(self: *Audio) !void {
const device = c.SDL_OpenAudioDevice(c.SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK, &spec);
if (device == 0) return error.OpenAudio;
self.device = device;
+ var format: c.SDL_AudioSpec = undefined;
+ var frames: c_int = 0;
+ if (c.SDL_GetAudioDeviceFormat(device, &format, &frames) and frames > 0 and format.freq > 0) {
+ self.pull = @intCast(@divFloor(@as(i64, frames) * sample_rate + format.freq - 1, format.freq));
+ }
self.voices_out = try self.bind();
self.scope.attach(device) catch {};
}
@@ -176,9 +202,18 @@ pub const Audio = struct {
if (clear) {
_ = c.SDL_ClearAudioStream(s.out);
s.pushed = 0;
+ s.was_fed = false;
}
if (samples.len == 0) return;
+ // Fed last frame and empty now: the device came up short since
+ // (counted at the push, as a pull before it would hide it by the
+ // next frame).
+ if (s.was_fed and !s.fed and c.SDL_GetAudioStreamQueued(s.out) == 0) {
+ s.underruns += 1;
+ self.dry = true;
+ }
if (c.SDL_PutAudioStreamData(s.out, samples.ptr, @intCast(samples.len * @sizeOf(f32)))) s.pushed += samples.len;
+ s.fed = true;
}
fn streamFor(self: *Audio, key: u32) ?*Stream {
@@ -192,14 +227,22 @@ pub const Audio = struct {
return &self.streams.items[self.streams.items.len - 1];
}
- /// Drops undeclared voices, keeps the voice mix queued ahead, and opens
- /// or closes the microphone.
+ /// Drops undeclared voices, works out the lead for the next frame, keeps
+ /// the voice mix queued ahead, and opens or closes the microphone.
pub fn endFrame(self: *Audio) void {
self.reconcileMicrophone();
var i: usize = 0;
while (i < self.voices.items.len) {
if (self.voices.items[i].seen) i += 1 else _ = self.voices.swapRemove(i);
}
+ const now = c.SDL_GetTicksNS();
+ if (self.last_frame != 0) {
+ self.intervals[self.frame % self.intervals.len] = now -| self.last_frame;
+ self.frame += 1;
+ }
+ self.last_frame = now;
+ self.lead = lead(self.intervals, self.pull, &self.margin, self.dry);
+ self.dry = false;
const out = self.voices_out orelse return;
const queued = c.SDL_GetAudioStreamQueued(out);
if (queued < 0) return;
@@ -257,16 +300,33 @@ pub const Audio = struct {
return false;
}
+ /// The streams as a frame starts.
pub fn streamStatus(self: *Audio) []const abi.Stream {
self.status.clearRetainingCapacity();
- for (self.streams.items) |s| {
- const queued = c.SDL_GetAudioStreamQueued(s.out);
- self.status.append(self.gpa, .{ .key = s.key, .queued = @intCast(@divFloor(@max(queued, 0), @sizeOf(f32))), .pushed = s.pushed }) catch {};
+ for (self.streams.items) |*s| {
+ const queued: u32 = @intCast(@divFloor(@max(c.SDL_GetAudioStreamQueued(s.out), 0), @sizeOf(f32)));
+ s.was_fed = s.fed;
+ s.fed = false;
+ self.status.append(self.gpa, .{ .key = s.key, .queued = queued, .pushed = s.pushed, .underruns = s.underruns }) catch {};
}
return self.status.items;
}
};
+/// How many samples to keep queued: the slowest of the last frames' worth
+/// (pushes come once a frame, `intervals` apart), plus the device's pull
+/// (it takes that much at once), plus `margin`: half a pull more at each
+/// underrun (`dry`), back down by a sample a frame.
+fn lead(intervals: [64]u64, pull: u32, margin: *u32, dry: bool) u32 {
+ var slowest: u64 = 0;
+ for (intervals) |ns| slowest = @max(slowest, ns);
+ margin.* = if (dry) @min(margin.* + pull / 2, max_lead) else margin.* -| 1;
+ const frame: u32 = @intCast(@min(slowest * sample_rate / std.time.ns_per_s, 1 << 20));
+ return @min(frame + pull + margin.*, max_lead);
+}
+/// 170 ms: a frame slower than that is a stall, not a pace.
+const max_lead = 8192;
+
/// Where the voice mix stops being passed through: -2.5 dBFS. Quieter
/// mixes keep their true volume to the bit; a clip normalised to full scale
/// only has its peaks bent (1 comes out at 0.875).
@@ -462,3 +522,20 @@ test "voices at another rate are resampled linearly, and a group's gain scales i
try std.testing.expect(out[0] > 0.375 and out[0] < 0.38);
try std.testing.expectEqual(@as(f32, 0.5), out[1023]);
}
+
+test "the lead the mixer gives streams outlasts the slowest recent frame and the device's pull, more after an underrun" {
+ const expectEqual = std.testing.expectEqual;
+ var intervals: [64]u64 = @splat(20 * std.time.ns_per_ms);
+ var margin: u32 = 0;
+ // 50 Hz and a pull of 1024: a frame's 960 samples and the pull.
+ try expectEqual(@as(u32, 960 + 1024), lead(intervals, 1024, &margin, false));
+ // One 50 ms frame in the window sets it.
+ intervals[5] = 50 * std.time.ns_per_ms;
+ try expectEqual(@as(u32, 2400 + 1024), lead(intervals, 1024, &margin, false));
+ // An underrun adds half a pull, which goes a sample a frame.
+ try expectEqual(@as(u32, 2400 + 1024 + 512), lead(intervals, 1024, &margin, true));
+ try expectEqual(@as(u32, 2400 + 1024 + 511), lead(intervals, 1024, &margin, false));
+ // A stall is not a pace: the lead stops at 170 ms.
+ intervals[6] = std.time.ns_per_s;
+ try expectEqual(@as(u32, max_lead), lead(intervals, 1024, &margin, false));
+}
diff --git a/src/sketch/root.zig b/src/sketch/root.zig
index d15046d..0ab04a7 100644
--- a/src/sketch/root.zig
+++ b/src/sketch/root.zig
@@ -219,6 +219,13 @@ pub const Frame = struct {
pub fn audio(f: *Frame, key: u32, samples: []const f32, options: struct { clear: bool = false }) void {
f.emit(.{ .audio = .{ .key = key, .samples = samples, .clear = options.clear } });
}
+ /// Streams `count` samples a compute shader wrote to `buffer` (f32 from
+ /// `offset`) on stream `key`, as `audio` streams samples: a GPU synth's
+ /// output, heard without a readback of the sketch's own. Make
+ /// `in.lead -| in.stream(key).queued` of them a frame.
+ pub fn audioBuffer(f: *Frame, key: u32, buffer: abi.Buffer, options: struct { offset: u32 = 0, count: u32 }) void {
+ f.emit(.{ .audio_buffer = .{ .key = key, .buffer = buffer, .offset = options.offset, .count = options.count } });
+ }
pub fn pixels(f: *Frame, img: Image, w: u32, h: u32, rgba: []const u8, options: struct { premultiplied: bool = false }) void {
f.emit(.{ .pixels = .{ .image = img, .w = w, .h = h, .rgba = rgba, .premultiplied = options.premultiplied } });
}
diff --git a/src/sketch/web.zig b/src/sketch/web.zig
index 62a173a..a035117 100644
--- a/src/sketch/web.zig
+++ b/src/sketch/web.zig
@@ -48,6 +48,8 @@ const js = struct {
extern "gpu" fn download(id: u32, offset: u32, size: u32, key: u32) void;
extern "audio" fn rate() u32;
extern "audio" fn push(key: u32, ptr: [*]const f32, len: usize, clear: u32) void;
+ /// `count` samples from a GPU buffer, queued on arrival.
+ extern "audio" fn buffer(key: u32, id: u32, offset: u32, count: u32) void;
extern "audio" fn queued(key: u32) u32;
/// Keeps the microphone open; it answers through `sk_microphone`.
extern "audio" fn listen() void;
@@ -225,6 +227,10 @@ fn Driver(comptime App: type) type {
s.pushed += v.samples.len;
js.push(v.key, v.samples.ptr, v.samples.len, @intFromBool(v.clear));
},
+ .audio_buffer => |v| {
+ stream(v.key).pushed += v.count;
+ js.buffer(v.key, @backingInt(v.buffer), v.offset, v.count);
+ },
.microphone => js.listen(),
.upload => |v| _ = js.write(@backingInt(v.buffer), v.offset, v.bytes.ptr, v.bytes.len),
.compute => |v| {