//! Sound: decoded clips played by reconciled voices, raw streams the sketch //! fills itself, and the microphone, open while declared. The audio thread //! only ever sees SDL's own copies of the samples; sketch code and memory //! never run on it. const std = @import("std"); const c = @import("c"); const abi = @import("abi"); const memory = @import("memory.zig"); pub const sample_rate = 48_000; const spec: c.SDL_AudioSpec = .{ .format = c.SDL_AUDIO_F32, .channels = 1, .freq = sample_rate }; /// A device block plus the next frame; 512 samples can underrun at 60 Hz. const queue_samples = 2048; /// How far a group's gain moves a sample toward a new one: a full swing /// takes 10 ms, so a gain set in a step does not click. const gain_step = 1.0 / 480.0; pub const Audio = struct { gpa: std.mem.Allocator, device: c.SDL_AudioDeviceID = 0, /// Mixed voices. voices_out: ?*c.SDL_AudioStream = null, clips: std.ArrayList([]f32) = .empty, paths: std.StringHashMapUnmanaged(abi.Audio) = .empty, voices: std.ArrayList(Playing) = .empty, /// Each group's gain as declared this frame, and as the mix has got to. gains: [8]f32 = @splat(1), heard_gains: [8]f32 = @splat(1), streams: std.ArrayList(Stream) = .empty, status: std.ArrayList(abi.Stream) = .empty, /// Samples the device takes from each stream at once. pull: u32 = 1024, /// The wall-clock time (ns) between the ends of the last frames' /// commands, where pushes are done; and when the last ended. intervals: [64]u64 = @splat(0), frame: u64 = 0, last_frame: u64 = 0, /// Grown by underruns, shrinking back while there are none. margin: u32 = 0, /// A stream ran dry since the lead was last worked out. dry: bool = false, /// `Input.lead`: see `lead`. lead: u32 = 4800, scope: Scope = .{}, /// The default recording device, bound to this stream while declared. microphone: ?*c.SDL_AudioStream = null, listening: bool = false, /// Set when opening failed, so it is tried once per declaration. deaf: bool = false, const Playing = struct { voice: abi.Voice, /// Where in the sound, in its samples: fractional when `rate` is. cursor: f64 = 0, done: bool = false, seen: bool = true, /// Mixed since a script last asked (`expect-audio`). sounded: bool = false, }; const Stream = struct { key: u32, out: *c.SDL_AudioStream, pushed: u64 = 0, /// Pushed to this frame, and the last. fed: bool = false, was_fed: bool = false, underruns: u32 = 0, }; pub fn open(self: *Audio) !void { const device = c.SDL_OpenAudioDevice(c.SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK, &spec); if (device == 0) return error.OpenAudio; self.device = device; var format: c.SDL_AudioSpec = undefined; var frames: c_int = 0; if (c.SDL_GetAudioDeviceFormat(device, &format, &frames) and frames > 0 and format.freq > 0) { self.pull = @intCast(@divFloor(@as(i64, frames) * sample_rate + format.freq - 1, format.freq)); } self.voices_out = try self.bind(); self.scope.attach(device) catch {}; } pub fn deinit(self: *Audio) void { if (self.microphone) |m| c.SDL_DestroyAudioStream(m); self.scope.deinit(); for (self.streams.items) |s| c.SDL_DestroyAudioStream(s.out); if (self.voices_out) |s| c.SDL_DestroyAudioStream(s); if (self.device != 0) c.SDL_CloseAudioDevice(self.device); for (self.clips.items) |samples| self.gpa.free(samples); var keys = self.paths.keyIterator(); while (keys.next()) |key| self.gpa.free(key.*); self.paths.deinit(self.gpa); self.clips.deinit(self.gpa); self.voices.deinit(self.gpa); self.streams.deinit(self.gpa); self.status.deinit(self.gpa); } fn bind(self: *Audio) !*c.SDL_AudioStream { const out = c.SDL_CreateAudioStream(&spec, &spec) orelse return error.OpenAudio; errdefer c.SDL_DestroyAudioStream(out); if (!c.SDL_BindAudioStream(self.device, out)) return error.OpenAudio; return out; } /// `.none` when the file is missing or unreadable, remembered so the /// warning prints once. pub fn load(self: *Audio, path: []const u8) abi.Audio { if (self.paths.get(path)) |audio| return audio; const audio: abi.Audio = if (decodeFile(self.gpa, path)) |samples| self.add(samples) else |err| blk: { std.log.warn("sound {s}: {t}", .{ path, err }); break :blk .none; }; const key = self.gpa.dupe(u8, path) catch return audio; self.paths.put(self.gpa, key, audio) catch self.gpa.free(key); return audio; } /// `.none` when `wav` does not decode. pub fn decode(self: *Audio, wav: []const u8) abi.Audio { return self.add(decodeBytes(self.gpa, wav) catch return .none); } /// A copy of `samples`; `.none` if one is not finite. pub fn new(self: *Audio, samples: []const f32) abi.Audio { for (samples) |v| if (!std.math.isFinite(v)) return .none; return self.add(self.gpa.dupe(f32, samples) catch return .none); } fn add(self: *Audio, samples: []f32) abi.Audio { self.clips.append(self.gpa, samples) catch { self.gpa.free(samples); return .none; }; return @fromBackingInt(@intCast(self.clips.items.len)); } pub fn samplesOf(self: *Audio, audio: abi.Audio) []const f32 { const index = @backingInt(audio); if (index == 0 or index > self.clips.items.len) return &.{}; return self.clips.items[index - 1]; } pub fn beginFrame(self: *Audio) void { for (self.voices.items) |*v| v.seen = false; self.gains = @splat(1); self.listening = false; } /// The microphone was declared this frame. pub fn listen(self: *Audio) void { self.listening = true; } /// What the microphone heard since the last call, into `arena`. pub fn record(self: *Audio, arena: std.mem.Allocator) []const f32 { const m = self.microphone orelse return &.{}; const available = c.SDL_GetAudioStreamAvailable(m); if (available <= 0) return &.{}; const samples = arena.alloc(f32, @as(usize, @intCast(available)) / @sizeOf(f32)) catch return &.{}; const got = c.SDL_GetAudioStreamData(m, samples.ptr, @intCast(samples.len * @sizeOf(f32))); if (got <= 0) return &.{}; return samples[0 .. @as(usize, @intCast(got)) / @sizeOf(f32)]; } /// Opens or closes the microphone to match this frame's declaration. fn reconcileMicrophone(self: *Audio) void { if (!self.listening) { if (self.microphone) |m| c.SDL_DestroyAudioStream(m); self.microphone = null; self.deaf = false; return; } if (self.microphone != null or self.deaf) return; const m = c.SDL_OpenAudioDeviceStream(c.SDL_AUDIO_DEVICE_DEFAULT_RECORDING, &spec, null, null) orelse { std.log.warn("microphone: {s}", .{c.SDL_GetError()}); self.deaf = true; return; }; if (!c.SDL_ResumeAudioStreamDevice(m)) std.log.warn("microphone: {s}", .{c.SDL_GetError()}); self.microphone = m; } /// Reconciles one declared voice with what is playing. pub fn declare(self: *Audio, voice: abi.Voice) void { for (self.voices.items) |*playing| { if (playing.voice.key != voice.key) continue; const restart = playing.voice.serial != voice.serial or playing.voice.audio != voice.audio; playing.voice = voice; playing.seen = true; if (restart) { playing.cursor = 0; playing.done = false; } return; } self.voices.append(self.gpa, .{ .voice = voice }) catch {}; } pub fn push(self: *Audio, key: u32, samples: []const f32, clear: bool) void { const s = self.streamFor(key) orelse return; if (clear) { _ = c.SDL_ClearAudioStream(s.out); s.pushed = 0; s.was_fed = false; } if (samples.len == 0) return; // Fed last frame and empty now: the device came up short since // (counted at the push, as a pull before it would hide it by the // next frame). if (s.was_fed and !s.fed and c.SDL_GetAudioStreamQueued(s.out) == 0) { s.underruns += 1; self.dry = true; } if (c.SDL_PutAudioStreamData(s.out, samples.ptr, @intCast(samples.len * @sizeOf(f32)))) s.pushed += samples.len; s.fed = true; } fn streamFor(self: *Audio, key: u32) ?*Stream { for (self.streams.items) |*s| if (s.key == key) return s; if (self.device == 0) return null; const out = self.bind() catch return null; self.streams.append(self.gpa, .{ .key = key, .out = out }) catch { c.SDL_DestroyAudioStream(out); return null; }; return &self.streams.items[self.streams.items.len - 1]; } /// Drops undeclared voices, works out the lead for the next frame, keeps /// the voice mix queued ahead, and opens or closes the microphone. pub fn endFrame(self: *Audio) void { self.reconcileMicrophone(); var i: usize = 0; while (i < self.voices.items.len) { if (self.voices.items[i].seen) i += 1 else _ = self.voices.swapRemove(i); } const now = c.SDL_GetTicksNS(); if (self.last_frame != 0) { self.intervals[self.frame % self.intervals.len] = now -| self.last_frame; self.frame += 1; } self.last_frame = now; self.lead = lead(self.intervals, self.pull, &self.margin, self.dry); self.dry = false; const out = self.voices_out orelse return; const queued = c.SDL_GetAudioStreamQueued(out); if (queued < 0) return; const needed = queue_samples -| @as(usize, @intCast(queued)) / @sizeOf(f32); var buffer: [queue_samples]f32 = undefined; const count = self.mix(buffer[0..needed]); if (count != 0) _ = c.SDL_PutAudioStreamData(out, &buffer, @intCast(count * @sizeOf(f32))); } /// Mixes live voices into `output`; returns how many samples were written, /// stopping early once every voice has finished. A voice at another /// rate is resampled by linear interpolation between its neighbours /// (at rate 1 it is its samples, exactly). pub fn mix(self: *Audio, output: []f32) usize { var count: usize = 0; for (output) |*sample| { var sum: f32 = 0; var alive = false; for (&self.heard_gains, self.gains) |*heard, gain| heard.* += std.math.clamp(gain - heard.*, -gain_step, gain_step); for (self.voices.items) |*playing| { if (playing.done) continue; const samples = self.samplesOf(playing.voice.audio); const len: f64 = @floatFromInt(samples.len); // Also ends a cursor an infinite or NaN rate made. if (!(playing.cursor < len)) { playing.done = true; continue; } alive = true; playing.sounded = true; const v = playing.voice; const i: usize = @intFromFloat(playing.cursor); const t: f32 = @floatCast(playing.cursor - @floor(playing.cursor)); const next = if (i + 1 < samples.len) samples[i + 1] else if (v.loop) samples[0] else 0; const at = if (t == 0) samples[i] else samples[i] + (next - samples[i]) * t; const edge = @min(i + 1, samples.len - i); const fade: f32 = if (v.fade == 0) 1 else @as(f32, @floatFromInt(@min(edge, v.fade))) / @as(f32, @floatFromInt(v.fade)); sum += at * v.volume * self.heard_gains[v.group] * fade; playing.cursor += @max(v.rate, 0); if (playing.cursor >= len and v.loop) playing.cursor = @mod(playing.cursor, len); } if (!alive) break; sample.* = limit(sum); count += 1; } return count; } /// Whether voice `key` was mixed since the last call: for scripts. pub fn sounded(self: *Audio, key: u32) bool { for (self.voices.items) |*playing| if (playing.voice.key == key) { defer playing.sounded = false; return playing.sounded; }; return false; } /// The streams as a frame starts. pub fn streamStatus(self: *Audio) []const abi.Stream { self.status.clearRetainingCapacity(); for (self.streams.items) |*s| { const queued: u32 = @intCast(@divFloor(@max(c.SDL_GetAudioStreamQueued(s.out), 0), @sizeOf(f32))); s.was_fed = s.fed; s.fed = false; self.status.append(self.gpa, .{ .key = s.key, .queued = queued, .pushed = s.pushed, .underruns = s.underruns }) catch {}; } return self.status.items; } }; /// How many samples to keep queued: the slowest of the last frames' worth /// (pushes come once a frame, `intervals` apart), plus the device's pull /// (it takes that much at once), plus `margin`: half a pull more at each /// underrun (`dry`), back down by a sample a frame. fn lead(intervals: [64]u64, pull: u32, margin: *u32, dry: bool) u32 { var slowest: u64 = 0; for (intervals) |ns| slowest = @max(slowest, ns); margin.* = if (dry) @min(margin.* + pull / 2, max_lead) else margin.* -| 1; const frame: u32 = @intCast(@min(slowest * sample_rate / std.time.ns_per_s, 1 << 20)); return @min(frame + pull + margin.*, max_lead); } /// 170 ms: a frame slower than that is a stall, not a pace. const max_lead = 8192; /// Where the voice mix stops being passed through: -2.5 dBFS. Quieter /// mixes keep their true volume to the bit; a clip normalised to full scale /// only has its peaks bent (1 comes out at 0.875). pub const knee = 0.75; /// The voice mix, bounded however many voices sound at once: unity gain up /// to the knee, then `knee + (1 - knee)·u/(1 + u)` with `u` how far past the /// knee in units of the room left, which meets the line at the knee with /// its slope (1) and tends to 1. (With a knee of 0 it is the old /// `x/(1 + |x|)`.) Written `1 - 1/(1 + u)` so that every step rounds /// monotonically and infinity comes out at 1. No state, no lookahead. pub fn limit(x: f32) f32 { const a = @abs(x); if (a <= knee) return x; const u = (a - knee) / (1 - knee); return std.math.copysign(knee + (1 - knee) * (1 - 1 / (1 + u)), x); } /// WAV, FLAC or Ogg Vorbis, by content (see `vendor/codecs.h`). fn decodeBytes(gpa: std.mem.Allocator, bytes: []const u8) ![]f32 { var channels: u32 = 0; var rate: u32 = 0; var frames: u64 = 0; const raw = c.sketchbook_decode_audio(bytes.ptr, bytes.len, &channels, &rate, &frames) orelse return error.DecodeAudio; defer c.sketchbook_free_audio(raw); if (rate == 0 or rate > 192000 or channels == 0 or channels > 8 or frames == 0 or frames > @as(u64, rate) * 600) return error.UnsupportedSoundFormat; const from: c.SDL_AudioSpec = .{ .format = c.SDL_AUDIO_F32, .channels = @intCast(channels), .freq = @intCast(rate) }; return convert(gpa, &from, std.mem.sliceAsBytes(raw[0..@intCast(frames * channels)])); } fn decodeFile(gpa: std.mem.Allocator, path: []const u8) ![]f32 { const bytes = memory.mapReadOnly(path); if (bytes.len == 0) return error.OpenSoundFile; defer std.posix.munmap(bytes); return decodeBytes(gpa, bytes); } /// To host format: f32 mono at `sample_rate`, all finite. fn convert(gpa: std.mem.Allocator, from: *const c.SDL_AudioSpec, data: []const u8) ![]f32 { if (data.len == 0 or data.len > std.math.maxInt(c_int)) return error.InvalidAudioLength; var out: [*c]u8 = null; var out_len: c_int = 0; if (!c.SDL_ConvertAudioSamples(from, data.ptr, @intCast(data.len), &spec, &out, &out_len)) return error.ConvertAudio; defer c.SDL_free(out); if (out_len <= 0 or @mod(out_len, 4) != 0) return error.InvalidAudioLength; const samples = @as([*]const f32, @ptrCast(@alignCast(out)))[0..@intCast(@divExact(out_len, 4))]; for (samples) |v| if (!std.math.isFinite(v)) return error.InvalidAudioSample; return gpa.dupe(f32, samples); } /// A read-only tap of the device's final mix, for the sound overlay. pub const Scope = struct { mutex: ?*c.SDL_Mutex = null, device: c.SDL_AudioDeviceID = 0, ring: [512]f32 = @splat(0), cursor: usize = 0, rate: u32 = sample_rate, pub fn attach(self: *Scope, device: c.SDL_AudioDeviceID) !void { self.mutex = c.SDL_CreateMutex() orelse return error.ScopeMutex; if (!c.SDL_SetAudioPostmixCallback(device, callback, self)) return error.ScopeCallback; self.device = device; } pub fn deinit(self: *Scope) void { // SDL waits for a callback in flight before removing it. if (self.device != 0) _ = c.SDL_SetAudioPostmixCallback(self.device, null, null); if (self.mutex) |m| c.SDL_DestroyMutex(m); self.* = .{}; } /// Oldest first. pub fn snapshot(self: *Scope) [512]f32 { var result: [512]f32 = @splat(0); const m = self.mutex orelse return result; c.SDL_LockMutex(m); defer c.SDL_UnlockMutex(m); for (&result, 0..) |*s, i| s.* = self.ring[(self.cursor + i) % self.ring.len]; return result; } fn callback(userdata: ?*anyopaque, s: [*c]const c.SDL_AudioSpec, buffer: [*c]f32, bytes: c_int) callconv(.c) void { const self: *Scope = @ptrCast(@alignCast(userdata orelse return)); if (s == null or s.*.channels <= 0 or bytes <= 0) return; const m = self.mutex orelse return; // Dropping one picture beats blocking the audio thread. if (!c.SDL_TryLockMutex(m)) return; defer c.SDL_UnlockMutex(m); const channels: usize = @intCast(s.*.channels); const frames = @as(usize, @intCast(bytes)) / @sizeOf(f32) / channels; self.rate = @intCast(s.*.freq); for (0..frames) |i| { self.ring[self.cursor] = buffer[i * channels]; self.cursor = (self.cursor + 1) % self.ring.len; } } }; test "voices reconcile: one-shots finish and stay silent, new serials restart, undeclared keys stop" { var audio: Audio = .{ .gpa = std.testing.allocator }; defer audio.deinit(); const clip = audio.add(try std.testing.allocator.dupe(f32, &.{ 0.5, 0.5, 0.5 })); var out: [8]f32 = undefined; audio.beginFrame(); audio.declare(.{ .key = 1, .audio = clip }); try std.testing.expectEqual(@as(usize, 3), audio.mix(&out)); audio.beginFrame(); audio.declare(.{ .key = 1, .audio = clip }); try std.testing.expectEqual(@as(usize, 0), audio.mix(&out)); audio.beginFrame(); audio.declare(.{ .key = 1, .audio = clip, .serial = 1 }); try std.testing.expectEqual(@as(usize, 3), audio.mix(&out)); audio.beginFrame(); audio.declare(.{ .key = 2, .audio = clip, .loop = true }); audio.endFrame(); try std.testing.expectEqual(@as(usize, 1), audio.voices.items.len); try std.testing.expectEqual(@as(usize, 8), audio.mix(&out)); } test "voices play made sounds: copied, and refused when not finite" { var audio: Audio = .{ .gpa = std.testing.allocator }; defer audio.deinit(); var made = [_]f32{ 0.25, -0.25 }; const sound = audio.new(&made); made[0] = 1; try std.testing.expectEqualSlices(f32, &.{ 0.25, -0.25 }, audio.samplesOf(sound)); try std.testing.expectEqual(abi.Audio.none, audio.new(&.{ 0, std.math.nan(f32) })); try std.testing.expectEqual(abi.Audio.none, audio.new(&.{std.math.inf(f32)})); } test "the mixer passes the mix through below the knee and limits it to 1 above, rising" { // Identity to the bit: every 997th float from 0 to the knee, both signs. const knee_bits: u32 = @bitCast(@as(f32, knee)); var bits: u32 = 0; while (bits <= knee_bits) : (bits += 997) { const x: f32 = @bitCast(bits); try std.testing.expectEqual(@as(u32, @bitCast(x)), @as(u32, @bitCast(limit(x)))); try std.testing.expectEqual(@as(u32, @bitCast(-x)), @as(u32, @bitCast(limit(-x)))); } try std.testing.expectEqual(@as(f32, knee), limit(knee)); // Rising and at most 1, from the knee's neighbourhood (every float) to // infinity (every 101st), and odd. var last = limit(@bitCast(knee_bits - 100_000)); bits = knee_bits - 100_000; const inf_bits: u32 = @bitCast(std.math.inf(f32)); while (bits <= inf_bits) : (bits += if (bits < knee_bits + 100_000) 1 else 101) { const x: f32 = @bitCast(bits); const y = limit(x); try std.testing.expect(y >= last and y <= 1); try std.testing.expectEqual(-y, limit(-x)); last = y; } try std.testing.expectEqual(@as(f32, 1), limit(std.math.inf(f32))); // Slope 1 on both sides of the knee, and what a full-scale peak becomes. const h = 1e-3; try std.testing.expectApproxEqAbs(@as(f32, 1), (limit(knee + h) - limit(knee)) / h, 0.01); try std.testing.expectEqual(@as(f32, 0.875), limit(1)); } test "voices at another rate are resampled linearly, and a group's gain scales its voices smoothly" { var audio: Audio = .{ .gpa = std.testing.allocator }; defer audio.deinit(); const clip = audio.new(&.{ 0, 0.1, 0.2, 0.3 }); var out: [1024]f32 = undefined; // Half speed: twice as long, the halves interpolated (the last toward // silence); double speed: every other sample. audio.declare(.{ .key = 1, .audio = clip, .rate = 0.5 }); try std.testing.expectEqual(@as(usize, 8), audio.mix(&out)); for ([_]f32{ 0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.15 }, out[0..8]) |want, got| try std.testing.expectApproxEqAbs(want, got, 1e-6); audio.declare(.{ .key = 1, .audio = clip, .rate = 2, .serial = 1 }); try std.testing.expectEqual(@as(usize, 2), audio.mix(&out)); try std.testing.expectEqualSlices(f32, &.{ 0, 0.2 }, out[0..2]); // Rate 0 holds it; an infinite rate ends it. audio.declare(.{ .key = 1, .audio = clip, .rate = 0, .serial = 2 }); try std.testing.expectEqual(@as(usize, 3), audio.mix(out[0..3])); audio.declare(.{ .key = 1, .audio = clip, .rate = std.math.inf(f32), .serial = 3 }); try std.testing.expectEqual(@as(usize, 1), audio.mix(&out)); // A looping voice in group 1 and one in group 0, at 0.25 each: group 1 // set to 0.5 glides there in 10 ms and stays; group 0 is untouched. const level = audio.new(&.{0.25}); audio.beginFrame(); audio.declare(.{ .key = 1, .audio = level, .loop = true, .group = 1 }); audio.declare(.{ .key = 2, .audio = level, .loop = true }); audio.gains[1] = 0.5; try std.testing.expectEqual(out.len, audio.mix(&out)); try std.testing.expect(out[0] < 0.5 and out[0] > 0.49); try std.testing.expect(out[100] > out[200] and out[200] > out[400]); try std.testing.expectEqual(@as(f32, 0.375), out[480]); try std.testing.expectEqual(@as(f32, 0.375), out[1023]); // Undeclared next frame: back to 1, as smoothly. audio.beginFrame(); audio.declare(.{ .key = 1, .audio = level, .loop = true, .group = 1 }); audio.declare(.{ .key = 2, .audio = level, .loop = true }); try std.testing.expectEqual(out.len, audio.mix(&out)); try std.testing.expect(out[0] > 0.375 and out[0] < 0.38); try std.testing.expectEqual(@as(f32, 0.5), out[1023]); } test "the lead the mixer gives streams outlasts the slowest recent frame and the device's pull, more after an underrun" { const expectEqual = std.testing.expectEqual; var intervals: [64]u64 = @splat(20 * std.time.ns_per_ms); var margin: u32 = 0; // 50 Hz and a pull of 1024: a frame's 960 samples and the pull. try expectEqual(@as(u32, 960 + 1024), lead(intervals, 1024, &margin, false)); // One 50 ms frame in the window sets it. intervals[5] = 50 * std.time.ns_per_ms; try expectEqual(@as(u32, 2400 + 1024), lead(intervals, 1024, &margin, false)); // An underrun adds half a pull, which goes a sample a frame. try expectEqual(@as(u32, 2400 + 1024 + 512), lead(intervals, 1024, &margin, true)); try expectEqual(@as(u32, 2400 + 1024 + 511), lead(intervals, 1024, &margin, false)); // A stall is not a pace: the lead stops at 170 ms. intervals[6] = std.time.ns_per_s; try expectEqual(@as(u32, max_lead), lead(intervals, 1024, &margin, false)); }