//! Sound: decoded clips played by reconciled voices, raw streams the sketch //! fills itself, and the microphone, open while declared. The audio thread //! only ever sees SDL's own copies of the samples; sketch code and memory //! never run on it. const std = @import("std"); const c = @import("c"); const abi = @import("abi"); const memory = @import("memory.zig"); pub const sample_rate = 48_000; const spec: c.SDL_AudioSpec = .{ .format = c.SDL_AUDIO_F32, .channels = 1, .freq = sample_rate }; /// A device block plus the next frame; 512 samples can underrun at 60 Hz. const queue_samples = 2048; pub const Audio = struct { gpa: std.mem.Allocator, device: c.SDL_AudioDeviceID = 0, /// Mixed voices. voices_out: ?*c.SDL_AudioStream = null, clips: std.ArrayList([]f32) = .empty, paths: std.StringHashMapUnmanaged(abi.Audio) = .empty, voices: std.ArrayList(Playing) = .empty, streams: std.ArrayList(Stream) = .empty, status: std.ArrayList(abi.Stream) = .empty, scope: Scope = .{}, /// The default recording device, bound to this stream while declared. microphone: ?*c.SDL_AudioStream = null, listening: bool = false, /// Set when opening failed, so it is tried once per declaration. deaf: bool = false, const Playing = struct { voice: abi.Voice, cursor: usize = 0, done: bool = false, seen: bool = true }; const Stream = struct { key: u32, out: *c.SDL_AudioStream, pushed: u64 = 0 }; pub fn open(self: *Audio) !void { const device = c.SDL_OpenAudioDevice(c.SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK, &spec); if (device == 0) return error.OpenAudio; self.device = device; self.voices_out = try self.bind(); self.scope.attach(device) catch {}; } pub fn deinit(self: *Audio) void { if (self.microphone) |m| c.SDL_DestroyAudioStream(m); self.scope.deinit(); for (self.streams.items) |s| c.SDL_DestroyAudioStream(s.out); if (self.voices_out) |s| c.SDL_DestroyAudioStream(s); if (self.device != 0) c.SDL_CloseAudioDevice(self.device); for (self.clips.items) |samples| self.gpa.free(samples); var keys = self.paths.keyIterator(); while (keys.next()) |key| self.gpa.free(key.*); self.paths.deinit(self.gpa); self.clips.deinit(self.gpa); self.voices.deinit(self.gpa); self.streams.deinit(self.gpa); self.status.deinit(self.gpa); } fn bind(self: *Audio) !*c.SDL_AudioStream { const out = c.SDL_CreateAudioStream(&spec, &spec) orelse return error.OpenAudio; errdefer c.SDL_DestroyAudioStream(out); if (!c.SDL_BindAudioStream(self.device, out)) return error.OpenAudio; return out; } /// `.none` when the file is missing or unreadable, remembered so the /// warning prints once. pub fn load(self: *Audio, path: []const u8) abi.Audio { if (self.paths.get(path)) |audio| return audio; const audio: abi.Audio = if (decodeFile(self.gpa, path)) |samples| self.add(samples) else |err| blk: { std.log.warn("sound {s}: {t}", .{ path, err }); break :blk .none; }; const key = self.gpa.dupe(u8, path) catch return audio; self.paths.put(self.gpa, key, audio) catch self.gpa.free(key); return audio; } /// `.none` when `wav` does not decode. pub fn decode(self: *Audio, wav: []const u8) abi.Audio { return self.add(decodeBytes(self.gpa, wav) catch return .none); } fn add(self: *Audio, samples: []f32) abi.Audio { self.clips.append(self.gpa, samples) catch { self.gpa.free(samples); return .none; }; return @fromBackingInt(@intCast(self.clips.items.len)); } pub fn samplesOf(self: *Audio, audio: abi.Audio) []const f32 { const index = @backingInt(audio); if (index == 0 or index > self.clips.items.len) return &.{}; return self.clips.items[index - 1]; } pub fn beginFrame(self: *Audio) void { for (self.voices.items) |*v| v.seen = false; self.listening = false; } /// The microphone was declared this frame. pub fn listen(self: *Audio) void { self.listening = true; } /// What the microphone heard since the last call, into `arena`. pub fn record(self: *Audio, arena: std.mem.Allocator) []const f32 { const m = self.microphone orelse return &.{}; const available = c.SDL_GetAudioStreamAvailable(m); if (available <= 0) return &.{}; const samples = arena.alloc(f32, @as(usize, @intCast(available)) / @sizeOf(f32)) catch return &.{}; const got = c.SDL_GetAudioStreamData(m, samples.ptr, @intCast(samples.len * @sizeOf(f32))); if (got <= 0) return &.{}; return samples[0 .. @as(usize, @intCast(got)) / @sizeOf(f32)]; } /// Opens or closes the microphone to match this frame's declaration. fn reconcileMicrophone(self: *Audio) void { if (!self.listening) { if (self.microphone) |m| c.SDL_DestroyAudioStream(m); self.microphone = null; self.deaf = false; return; } if (self.microphone != null or self.deaf) return; const m = c.SDL_OpenAudioDeviceStream(c.SDL_AUDIO_DEVICE_DEFAULT_RECORDING, &spec, null, null) orelse { std.log.warn("microphone: {s}", .{c.SDL_GetError()}); self.deaf = true; return; }; if (!c.SDL_ResumeAudioStreamDevice(m)) std.log.warn("microphone: {s}", .{c.SDL_GetError()}); self.microphone = m; } /// Reconciles one declared voice with what is playing. pub fn declare(self: *Audio, voice: abi.Voice) void { for (self.voices.items) |*playing| { if (playing.voice.key != voice.key) continue; const restart = playing.voice.serial != voice.serial or playing.voice.audio != voice.audio; playing.voice = voice; playing.seen = true; if (restart) { playing.cursor = 0; playing.done = false; } return; } self.voices.append(self.gpa, .{ .voice = voice }) catch {}; } pub fn push(self: *Audio, key: u32, samples: []const f32, clear: bool) void { const s = self.streamFor(key) orelse return; if (clear) { _ = c.SDL_ClearAudioStream(s.out); s.pushed = 0; } if (samples.len == 0) return; if (c.SDL_PutAudioStreamData(s.out, samples.ptr, @intCast(samples.len * @sizeOf(f32)))) s.pushed += samples.len; } fn streamFor(self: *Audio, key: u32) ?*Stream { for (self.streams.items) |*s| if (s.key == key) return s; if (self.device == 0) return null; const out = self.bind() catch return null; self.streams.append(self.gpa, .{ .key = key, .out = out }) catch { c.SDL_DestroyAudioStream(out); return null; }; return &self.streams.items[self.streams.items.len - 1]; } /// Drops undeclared voices, keeps the voice mix queued ahead, and opens /// or closes the microphone. pub fn endFrame(self: *Audio) void { self.reconcileMicrophone(); var i: usize = 0; while (i < self.voices.items.len) { if (self.voices.items[i].seen) i += 1 else _ = self.voices.swapRemove(i); } const out = self.voices_out orelse return; const queued = c.SDL_GetAudioStreamQueued(out); if (queued < 0) return; const needed = queue_samples -| @as(usize, @intCast(queued)) / @sizeOf(f32); var buffer: [queue_samples]f32 = undefined; const count = self.mix(buffer[0..needed]); if (count != 0) _ = c.SDL_PutAudioStreamData(out, &buffer, @intCast(count * @sizeOf(f32))); } /// Mixes live voices into `output`; returns how many samples were written, /// stopping early once every voice has finished. pub fn mix(self: *Audio, output: []f32) usize { var count: usize = 0; for (output) |*sample| { var sum: f32 = 0; var alive = false; for (self.voices.items) |*playing| { if (playing.done) continue; const samples = self.samplesOf(playing.voice.audio); if (playing.cursor >= samples.len) { playing.done = true; continue; } alive = true; const v = playing.voice; const edge = @min(playing.cursor + 1, samples.len - playing.cursor); const fade: f32 = if (v.fade == 0) 1 else @as(f32, @floatFromInt(@min(edge, v.fade))) / @as(f32, @floatFromInt(v.fade)); sum += samples[playing.cursor] * v.volume * fade; playing.cursor += 1; if (playing.cursor == samples.len and v.loop) playing.cursor = 0; } if (!alive) break; // Bounded however many voices sound at once. sample.* = sum / (1 + @abs(sum)); count += 1; } return count; } pub fn streamStatus(self: *Audio) []const abi.Stream { self.status.clearRetainingCapacity(); for (self.streams.items) |s| { const queued = c.SDL_GetAudioStreamQueued(s.out); self.status.append(self.gpa, .{ .key = s.key, .queued = @intCast(@divFloor(@max(queued, 0), @sizeOf(f32))), .pushed = s.pushed }) catch {}; } return self.status.items; } }; /// WAV, FLAC or Ogg Vorbis, by content (see `vendor/codecs.h`). fn decodeBytes(gpa: std.mem.Allocator, bytes: []const u8) ![]f32 { var channels: u32 = 0; var rate: u32 = 0; var frames: u64 = 0; const raw = c.sketchbook_decode_audio(bytes.ptr, bytes.len, &channels, &rate, &frames) orelse return error.DecodeAudio; defer c.sketchbook_free_audio(raw); if (rate == 0 or rate > 192000 or channels == 0 or channels > 8 or frames == 0 or frames > @as(u64, rate) * 600) return error.UnsupportedSoundFormat; const from: c.SDL_AudioSpec = .{ .format = c.SDL_AUDIO_F32, .channels = @intCast(channels), .freq = @intCast(rate) }; return convert(gpa, &from, std.mem.sliceAsBytes(raw[0..@intCast(frames * channels)])); } fn decodeFile(gpa: std.mem.Allocator, path: []const u8) ![]f32 { const bytes = memory.mapReadOnly(path); if (bytes.len == 0) return error.OpenSoundFile; defer std.posix.munmap(bytes); return decodeBytes(gpa, bytes); } /// To host format: f32 mono at `sample_rate`, all finite. fn convert(gpa: std.mem.Allocator, from: *const c.SDL_AudioSpec, data: []const u8) ![]f32 { if (data.len == 0 or data.len > std.math.maxInt(c_int)) return error.InvalidAudioLength; var out: [*c]u8 = null; var out_len: c_int = 0; if (!c.SDL_ConvertAudioSamples(from, data.ptr, @intCast(data.len), &spec, &out, &out_len)) return error.ConvertAudio; defer c.SDL_free(out); if (out_len <= 0 or @mod(out_len, 4) != 0) return error.InvalidAudioLength; const samples = @as([*]const f32, @ptrCast(@alignCast(out)))[0..@intCast(@divExact(out_len, 4))]; for (samples) |v| if (!std.math.isFinite(v)) return error.InvalidAudioSample; return gpa.dupe(f32, samples); } /// A read-only tap of the device's final mix, for the sound overlay. pub const Scope = struct { mutex: ?*c.SDL_Mutex = null, device: c.SDL_AudioDeviceID = 0, ring: [512]f32 = @splat(0), cursor: usize = 0, rate: u32 = sample_rate, pub fn attach(self: *Scope, device: c.SDL_AudioDeviceID) !void { self.mutex = c.SDL_CreateMutex() orelse return error.ScopeMutex; if (!c.SDL_SetAudioPostmixCallback(device, callback, self)) return error.ScopeCallback; self.device = device; } pub fn deinit(self: *Scope) void { // SDL waits for a callback in flight before removing it. if (self.device != 0) _ = c.SDL_SetAudioPostmixCallback(self.device, null, null); if (self.mutex) |m| c.SDL_DestroyMutex(m); self.* = .{}; } /// Oldest first. pub fn snapshot(self: *Scope) [512]f32 { var result: [512]f32 = @splat(0); const m = self.mutex orelse return result; c.SDL_LockMutex(m); defer c.SDL_UnlockMutex(m); for (&result, 0..) |*s, i| s.* = self.ring[(self.cursor + i) % self.ring.len]; return result; } fn callback(userdata: ?*anyopaque, s: [*c]const c.SDL_AudioSpec, buffer: [*c]f32, bytes: c_int) callconv(.c) void { const self: *Scope = @ptrCast(@alignCast(userdata orelse return)); if (s == null or s.*.channels <= 0 or bytes <= 0) return; const m = self.mutex orelse return; // Dropping one picture beats blocking the audio thread. if (!c.SDL_TryLockMutex(m)) return; defer c.SDL_UnlockMutex(m); const channels: usize = @intCast(s.*.channels); const frames = @as(usize, @intCast(bytes)) / @sizeOf(f32) / channels; self.rate = @intCast(s.*.freq); for (0..frames) |i| { self.ring[self.cursor] = buffer[i * channels]; self.cursor = (self.cursor + 1) % self.ring.len; } } }; test "voices reconcile: one-shots finish and stay silent, new serials restart, undeclared keys stop" { var audio: Audio = .{ .gpa = std.testing.allocator }; defer audio.deinit(); const clip = audio.add(try std.testing.allocator.dupe(f32, &.{ 0.5, 0.5, 0.5 })); var out: [8]f32 = undefined; audio.beginFrame(); audio.declare(.{ .key = 1, .audio = clip }); try std.testing.expectEqual(@as(usize, 3), audio.mix(&out)); audio.beginFrame(); audio.declare(.{ .key = 1, .audio = clip }); try std.testing.expectEqual(@as(usize, 0), audio.mix(&out)); audio.beginFrame(); audio.declare(.{ .key = 1, .audio = clip, .serial = 1 }); try std.testing.expectEqual(@as(usize, 3), audio.mix(&out)); audio.beginFrame(); audio.declare(.{ .key = 2, .audio = clip, .loop = true }); audio.endFrame(); try std.testing.expectEqual(@as(usize, 1), audio.voices.items.len); try std.testing.expectEqual(@as(usize, 8), audio.mix(&out)); }