//! Live views of a stream of sound, analysed and drawn on the GPU: the last //! few seconds as a scrolling waveform over its spectrogram, the smoothed //! spectrum of the newest samples, and an oscilloscope. A sketch feeds a //! `Scope` (kept in its Session) with `push` and draws it once a frame with //! `draw`, giving the rectangles of the views it wants. //! //! The analyses (see the shaders for the details): //! - spectrogram: a 2048-point Blackman-Harris window per column, and below //! 500 Hz a second analysis decimated 4× (6 Hz resolution) averaged over //! three windows; log-frequency rows; //! - spectrum: a 4096-point window zero-padded to 8192, log-frequency //! columns, fast attack and slow release, falling peaks; //! - waveform: min, max and RMS per column, joined into a line when zoomed //! past one sample per column. //! //! Columns of the scrolling views are analysed once, when everything their //! windows reach has arrived, into rings the views read from the oldest; so //! the spectrogram runs `reach` samples (about 130 ms) behind the newest. const std = @import("std"); const sk = @import("sketch"); const gpu = sk.abi.gpu; pub const layouts = @import("scope_layouts"); const shaders = @import("scope_shaders"); pub const min_freq = 20.0; pub const floor_db = -96.0; /// The samples kept: enough for a frame's new columns and all they read. pub const history = 1 << 16; /// How far the spectrogram reads past a column's centre (its long /// analysis: 2048 points decimated 4×, three windows a quarter apart). pub const reach = 6160; pub const Tint = layouts.Tint; /// Where and how to draw; views left null are not analysed. pub const Views = struct { /// Seconds the waveform and spectrogram span, ending now. seconds: f32 = 6, /// Both scroll together, column for column: give them the same width. wave: ?sk.Rect = null, spectrogram: ?sk.Rect = null, spectrum: ?sk.Rect = null, oscilloscope: ?Oscilloscope = null, gain: f32 = 1, tint: Tint = .live, pub const Oscilloscope = struct { rect: sk.Rect, /// The samples across it. samples: u32 = 1024, /// How many samples before the newest it ends: a sketch steadies a /// periodic wave by ending it at the same phase each frame. end: u32 = 0, }; }; pub const Scope = struct { samples: [history]f32 = @splat(0), /// Samples pushed since the start. heard: u64 = 0, /// Pushed since the last `draw`, so the GPU copy is stale. fresh: bool = false, /// The next column to analyse, counted from the first sample; how many /// have been since the view was (re)made; their count and width. next: u64 = 0, filled: u32 = 0, columns: u32 = 0, spp: u32 = 0, gpu: Gpu = .{}, pub fn push(s: *Scope, heard: []const f32) void { if (heard.len == 0) return; const h = &s.samples; if (heard.len >= h.len) { @memcpy(h, heard[heard.len - h.len ..]); } else { std.mem.copyForwards(f32, h[0 .. h.len - heard.len], h[heard.len..]); @memcpy(h[h.len - heard.len ..], heard); } s.heard += heard.len; s.fresh = true; } /// Analyses what is new and draws the views; needs `f.host.gpu()`. pub fn draw(s: *Scope, f: *sk.Frame, v: Views) void { const g = &s.gpu; const rate: f64 = @floatFromInt(f.in.sample_rate); g.setup(f); if (f.in.reloaded) g.forget(); if (s.fresh) f.upload(g.tape, 0, std.mem.sliceAsBytes(&s.samples)); defer s.fresh = false; if (v.wave orelse v.spectrogram) |r| s.scroll(f, v, r, rate); if (v.spectrum) |r| s.spectrum(f, r, rate); if (v.oscilloscope) |o| s.oscilloscope(f, v, o); } /// The waveform and spectrogram, scrolling. fn scroll(s: *Scope, f: *sk.Frame, v: Views, r: sk.Rect, rate: f64) void { const g = &s.gpu; const columns: u32 = @intFromFloat(std.math.clamp(r.w, 1, layouts.max_columns)); const spp: u32 = @max(1, @as(u32, @intFromFloat(@round(v.seconds * rate / @as(f64, @floatFromInt(columns)))))); if (columns != s.columns or spp != s.spp or f.in.reloaded) { s.columns = columns; s.spp = spp; s.filled = 0; s.next = 0; } const rows: u32 = if (v.spectrogram) |sr| @intFromFloat(std.math.clamp(sr.h, 1, layouts.max_rows)) else 1; g.levelsFor(f, rows); // History index of sample `i`: `i - base`. A column is ready once // all it reads has arrived, and analysable while all it reads is kept. const base: i64 = @as(i64, @intCast(s.heard)) - history; const ready = (s.heard -| reach) / spp; const kept: u64 = @intCast(@max(0, @divFloor(base + reach + spp - 1, spp))); s.next = @max(s.next, kept, ready -| columns); if (ready > s.next) { const count: u32 = @intCast(ready - s.next); const first: u32 = @intCast(@as(i64, @intCast(s.next * spp)) - base); const slot: u32 = @intCast(s.next % columns); if (v.wave != null) f.compute(.{ .label = "scope waveform.comp", .kernel = waveform_kernel, .readonly = &.{g.tape}, .readwrite = &.{g.wave.columns}, .uniforms = gpu.uniform(f.scratch, layouts.WaveParams{ .offset = 0, .first = first, .samples = history, .columns = count, .spp = @floatFromInt(spp), .slot = slot, .ring = columns }), .groups = .{ count, 1, 1 }, }); if (v.spectrogram != null) f.compute(.{ .label = "scope spectrogram.comp", .kernel = spectrogram_kernel, .readonly = &.{g.tape}, .readwrite = &.{g.levels}, .uniforms = gpu.uniform(f.scratch, layouts.SpectrogramParams{ .offset = 0, .first = first, .samples = history, .columns = count, .rows = rows, .spp = @floatFromInt(spp), .rate = @floatCast(rate), .min_freq = min_freq, .max_freq = maxFreq(rate), .slot = slot, .ring = columns, }), .groups = .{ count, 1, 1 }, }); s.filled = @intCast(@min(columns, s.filled + count)); s.next = ready; } const scroll_at: u32 = @intCast(s.next % columns); if (v.wave) |wr| { const size = target(f, &g.wave, wr, layouts.max_rows); const view: layouts.WaveView = .{ .size = .{ @floatFromInt(size[0]), @floatFromInt(size[1]) }, .playhead = -1, .gain = v.gain, .tint = @intFromEnum(v.tint), .scroll = scroll_at, .filled = s.filled, }; if (g.wave.redraw(std.hash.Wyhash.hash(0, std.mem.asBytes(&view)))) wavePass(f, &g.wave, size, view); f.image(g.wave.image, wr, .{}); } if (v.spectrogram) |sr| { const size = target(f, &g.spectrogram, sr, rows); const view: layouts.SpectrogramView = .{ .size = .{ @floatFromInt(size[0]), @floatFromInt(size[1]) }, .floor_db = floor_db, .ceil_db = -6, .playhead = -1, .scroll = scroll_at, .filled = s.filled, }; if (g.spectrogram.redraw(std.hash.Wyhash.hash(0, std.mem.asBytes(&view)))) f.gpu(.{ .label = "scope spectrogram.frag", .image = g.spectrogram.image, .width = size[0], .height = size[1], .clear = null, .pipelines = &spectrogram_pipelines, .draws = &.{.{ .pipeline = 0, .vertex_count = 3, .fragment_uniforms = gpu.uniform(f.scratch, view), .fragment_storage = &.{g.levels} }}, }); f.image(g.spectrogram.image, sr, .{}); } } /// The spectrum of the newest samples, smoothed frame to frame. fn spectrum(s: *Scope, f: *sk.Frame, r: sk.Rect, rate: f64) void { const g = &s.gpu; const size = target(f, &g.spectrum, r, layouts.max_rows); const dt: f64 = @min(@as(f64, @floatFromInt(f.in.dt)) / std.time.ns_per_s, 0.1); f.compute(.{ .label = "scope spectrum.comp", .kernel = spectrum_kernel, .readonly = &.{g.tape}, .readwrite = &.{ g.scratch, g.spectrum.columns }, .uniforms = gpu.uniform(f.scratch, layouts.SpectrumParams{ .offset = 0, .end = history, .samples = history, .columns = size[0], .reset = @intFromBool(!g.spectrum_started), .rate = @floatCast(rate), .min_freq = min_freq, .max_freq = maxFreq(rate), .dt = @floatCast(dt), }), }); g.spectrum_started = true; f.gpu(.{ .label = "scope spectrum.frag", .image = g.spectrum.image, .width = size[0], .height = size[1], .clear = null, .pipelines = &spectrum_pipelines, .draws = &.{.{ .pipeline = 0, .vertex_count = 3, .fragment_uniforms = gpu.uniform(f.scratch, layouts.SpectrumView{ .size = .{ @floatFromInt(size[0]), @floatFromInt(size[1]) }, .floor_db = floor_db, .ceil_db = 0, .min_freq = min_freq, .max_freq = maxFreq(rate), }), .fragment_storage = &.{g.spectrum.columns}, }}, }); f.image(g.spectrum.image, r, .{}); } /// The newest `samples` (less `end`), as a line. fn oscilloscope(s: *Scope, f: *sk.Frame, v: Views, o: Views.Oscilloscope) void { const g = &s.gpu; const size = target(f, &g.oscilloscope, o.rect, layouts.max_rows); const n: u32 = @min(o.samples, history / 2); const end: u32 = @min(o.end, history / 2); const first: u32 = history - n - end; f.compute(.{ .label = "scope oscilloscope", .kernel = waveform_kernel, .readonly = &.{g.tape}, .readwrite = &.{g.oscilloscope.columns}, .uniforms = gpu.uniform(f.scratch, layouts.WaveParams{ .offset = 0, .first = first, .samples = history, .columns = size[0], .spp = @as(f32, @floatFromInt(n)) / @as(f32, @floatFromInt(size[0])), .slot = 0, .ring = layouts.max_columns, }), .groups = .{ size[0], 1, 1 }, }); wavePass(f, &g.oscilloscope, size, .{ .size = .{ @floatFromInt(size[0]), @floatFromInt(size[1]) }, .playhead = -1, .gain = v.gain, .tint = @intFromEnum(v.tint), .scroll = 0, .filled = size[0], }); f.image(g.oscilloscope.image, o.rect, .{}); } }; /// The highest frequency the views show. pub fn maxFreq(rate: f64) f32 { return @floatCast(@min(20_000, rate / 2)); } /// Where `hz` falls across a spectrum view (or up a spectrogram, from its /// bottom): 0 to 1, log frequency. pub fn place(hz: f64, rate: f64) f32 { return @floatCast(@log(hz / min_freq) / @log(maxFreq(rate) / min_freq)); } // --------------------------------------------------------------------------- // GPU objects. /// One view's image and columns, and the inputs it was last drawn from. const View = struct { image: sk.Image = .none, size: [2]u32 = .{ 0, 0 }, columns: sk.Buffer = .none, drawn: u64 = 0, fn redraw(v: *View, key: u64) bool { if (key == v.drawn) return false; v.drawn = key; return true; } }; const Gpu = struct { ready: bool = false, tape: sk.Buffer = .none, scratch: sk.Buffer = .none, levels: sk.Buffer = .none, levels_rows: u32 = 0, wave: View = .{}, spectrogram: View = .{}, spectrum: View = .{}, oscilloscope: View = .{}, spectrum_started: bool = false, fn setup(g: *Gpu, f: *sk.Frame) void { if (g.ready) return; g.ready = true; g.tape = f.host.newBuffer(history * @sizeOf(f32)); g.scratch = f.host.newBuffer(layouts.fft_size * 2 * @sizeOf([2]f32)); g.wave.columns = f.host.newBuffer(layouts.max_columns * @sizeOf([4]f32)); g.oscilloscope.columns = f.host.newBuffer(layouts.max_columns * @sizeOf([4]f32)); g.spectrum.columns = f.host.newBuffer(layouts.max_columns * @sizeOf([2]f32)); } /// The spectrogram's levels, `rows` of `max_columns`. fn levelsFor(g: *Gpu, f: *sk.Frame, rows: u32) void { if (rows <= g.levels_rows) return; if (g.levels != .none) f.host.freeBuffer(g.levels); g.levels = f.host.newBuffer(layouts.max_columns * rows * @sizeOf(f32)); g.levels_rows = rows; g.forget(); } /// New shaders, or new buffers: draw everything again. fn forget(g: *Gpu) void { g.wave.drawn = 0; g.spectrogram.drawn = 0; g.spectrum_started = false; } }; /// The view's image at `r`'s size, (re)made when that changes. fn target(f: *sk.Frame, v: *View, r: sk.Rect, max_h: u32) [2]u32 { const want: [2]u32 = .{ @intFromFloat(std.math.clamp(r.w, 1, layouts.max_columns)), @intFromFloat(std.math.clamp(r.h, 1, @as(f32, @floatFromInt(max_h)))), }; if (v.image == .none or v.size[0] != want[0] or v.size[1] != want[1]) { if (v.image != .none) f.host.freeImage(v.image); v.image = f.host.newImage(want[0], want[1]); v.size = want; v.drawn = 0; } return want; } fn wavePass(f: *sk.Frame, v: *View, size: [2]u32, view: layouts.WaveView) void { f.gpu(.{ .label = "scope waveform.frag", .image = v.image, .width = size[0], .height = size[1], .clear = null, .pipelines = &wave_pipelines, .draws = &.{.{ .pipeline = 0, .vertex_count = 3, .fragment_uniforms = gpu.uniform(f.scratch, view), .fragment_storage = &.{v.columns} }}, }); } fn viewPipeline(fragment: []const u8) gpu.Pipeline { return .{ .vertex = .{ .code = shaders.view_vert }, .fragment = .{ .code = fragment, .uniform_buffers = 1, .storage_buffers = 1 }, .stride = 0, .attributes = &.{}, .blend = .none, }; } const wave_pipelines = [_]gpu.Pipeline{viewPipeline(shaders.waveform_frag)}; const spectrum_pipelines = [_]gpu.Pipeline{viewPipeline(shaders.spectrum_frag)}; const spectrogram_pipelines = [_]gpu.Pipeline{viewPipeline(shaders.spectrogram_frag)}; const spectrum_kernel: gpu.Kernel = .{ .code = shaders.spectrum_comp, .readonly_buffers = 1, .readwrite_buffers = 2, .uniform_buffers = 1, .threads = .{ 256, 1, 1 } }; const spectrogram_kernel: gpu.Kernel = .{ .code = shaders.spectrogram_comp, .readonly_buffers = 1, .readwrite_buffers = 1, .uniform_buffers = 1, .threads = .{ 256, 1, 1 } }; const waveform_kernel: gpu.Kernel = .{ .code = shaders.waveform_comp, .readonly_buffers = 1, .readwrite_buffers = 1, .uniform_buffers = 1, .threads = .{ 64, 1, 1 } }; test "the history keeps the newest samples in order" { var s: Scope = .{}; s.push(&.{ 1, 2, 3 }); s.push(&.{4}); try std.testing.expectEqualSlices(f32, &.{ 1, 2, 3, 4 }, s.samples[history - 4 ..]); try std.testing.expectEqual(@as(u64, 4), s.heard); } test "frequencies place on a log axis" { try std.testing.expectEqual(@as(f32, 0), place(min_freq, 48_000)); try std.testing.expectApproxEqAbs(@as(f32, 1), place(20_000, 48_000), 1e-6); }