summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorskal <pascal.massimino@gmail.com>2026-02-07 15:07:01 +0100
committerskal <pascal.massimino@gmail.com>2026-02-07 15:07:01 +0100
commita0dd0a27c4d6831fb2fb5ad81283f36512ef16ef (patch)
treebc961189b10cf9f983d39854b9a5770d87574427
parenta6a7bf0440dbabdc6c994c0fb21a8ac31c27be07 (diff)
update doc, optimize spectral_editor
-rw-r--r--PROJECT_CONTEXT.md13
-rw-r--r--TODO.md19
-rw-r--r--tools/spectral_editor/script.js140
3 files changed, 81 insertions, 91 deletions
diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md
index e5adfb6..8f71ebf 100644
--- a/PROJECT_CONTEXT.md
+++ b/PROJECT_CONTEXT.md
@@ -32,6 +32,19 @@ Style:
### Recently Completed
+#### Milestone: Audio Peak Measurement & Test Coverage (February 7, 2026) ✅
+- **Real-Time Audio Peak Fix**: Resolved critical audio-visual synchronization bug where visual effects triggered ~400ms before audio was heard. Root cause: peak measurement occurred at ring buffer write time (synth_render) instead of playback time (audio callback). Solution: Added `get_realtime_peak()` to AudioBackend interface, implemented real-time peak tracking in MiniaudioBackend audio_callback using exponential averaging (instant attack, 0.7 decay rate for 1-second fade). Updated main.cc and test_demo.cc to use `audio_get_realtime_peak()` API. Result: Perfect audio-visual synchronization, visual effects trigger precisely when audio is heard.
+
+- **Peak Decay Optimization**: Fixed "test_demo just flashing" issue caused by slow decay rate. Old: 0.95 decay = 5.76 second fade (screen stayed white constantly). New: 0.7 decay = 1.15 second fade (proper flash with smooth decay). Mathematical verification: At 128ms audio callbacks, 0.7^9 ≈ 0.1 after ~1 second. Result: Responsive visual effects with proper attack/decay curves.
+
+- **SilentBackend Test Infrastructure**: Created test-only audio backend for comprehensive audio.cc testing without hardware. Features: controllable peak for edge cases, tracks frames rendered and voice triggers, pure inspection (no audio output). Created 7 comprehensive tests covering lifecycle, peak control, frame/voice tracking, playback time, buffer management, audio_update safety. Result: Significantly improved audio.cc test coverage (audio_get_playback_time, audio_render_ahead, audio_update all tested).
+
+- **Backend Reorganization**: Moved all audio backend implementations to src/audio/backend/ subdirectory for cleaner code organization. Moved: miniaudio_backend, mock_audio_backend, wav_dump_backend, jittered_audio_backend, silent_backend (5 backends total). Updated all #include paths in backend files and tests, updated CMakeLists.txt paths. Kept audio_backend.h (interface) in src/audio/. Result: Improved maintainability, zero functionality regressions, all 28 tests pass.
+
+- **Dead Code Removal**: Removed unused `register_spec_asset()` function from audio.{h,cc}. Function was never called anywhere in codebase. Removed declaration and implementation (lines 43-58). Result: Cleaner codebase, reduced maintenance burden.
+
+- **Test Coverage Achievement**: All 28 tests passing (100% success rate). New SilentBackend tests provide comprehensive coverage for audio subsystem. No regressions in existing tests. Build clean with no errors or warnings. Files modified: 15 changed, 3 new files, 8 files reorganized. Commit a6a7bf0 ready for push.
+
#### Milestone: test_demo - Audio/Visual Sync Debug Tool (February 7, 2026) 🎯
- **Standalone Debug Tool**: Created minimal test executable for debugging audio/visual synchronization and variable tempo system without full demo complexity. **Core Features**: Simple drum beat (kick-snare) with crash landmarks at bars 3 and 7, NOTE_A4 (440 Hz) reference tone at start of each bar for testing, screen flash effect synchronized to audio peaks, 16 second duration (8 bars at 120 BPM). **Variable Tempo Mode**: `--tempo` flag enables alternating tempo scaling (even bars: 1.0x → 1.5x acceleration, odd bars: 1.0x → 0.66x deceleration) to test music time vs physical time system. **Peak Logging**: `--log-peaks FILE` exports audio peak data for gnuplot visualization, `--log-peaks-fine` adds millisecond-resolution per-frame logging (~960 samples vs 32 beat-aligned samples). **Command-Line Options**: `--help` shows usage, `--fullscreen` for fullscreen mode, `--resolution WxH` for custom window size. **Error Handling**: Unknown options print error message and help text before exiting with status 1. **File Structure**: `src/test_demo.cc` (main executable, ~220 lines), `assets/test_demo.track` (drum patterns with NOTE_A4), `assets/test_demo.seq` (visual timeline), `test_demo_README.md` (comprehensive documentation). **Peak Log Format**: Beat-aligned mode logs at beat boundaries (32 samples), fine-grained mode logs every frame with beat_number column for correlation. **Build Integration**: CMake target with timeline/music generation, proper dependencies, size optimizations. **Use Cases**: Verify millisecond-precision sync, detect timing jitter, analyze tempo scaling effects, debug flash-to-audio alignment. **Impact**: Provides isolated testing environment for sync verification without demo64k complexity, enables data-driven analysis via exported logs.
diff --git a/TODO.md b/TODO.md
index 3a27d01..83f6d35 100644
--- a/TODO.md
+++ b/TODO.md
@@ -2,6 +2,25 @@
This file tracks prioritized tasks with detailed attack plans.
+## Recently Completed (February 7, 2026)
+
+- [x] **Audio Peak Measurement & Test Coverage Improvements** (February 7, 2026)
+ - [x] **Real-Time Peak Fix**: Fixed critical audio-visual sync bug where visual effects triggered ~400ms before audio was heard
+ - Root cause: Peak measured at ring buffer write time (synth_render) instead of playback time (audio callback)
+ - Solution: Added `get_realtime_peak()` to AudioBackend interface, implemented in MiniaudioBackend audio_callback
+ - Used exponential averaging: instant attack, 0.7 decay rate (1-second fade time)
+ - [x] **Peak Decay Optimization**: Fixed "test_demo just flashing" issue
+ - Old: 0.95 decay = 5.76 second fade (screen stayed white)
+ - New: 0.7 decay = 1.15 second fade (proper flash with smooth decay)
+ - [x] **SilentBackend Creation**: Test-only backend for audio.cc testing without hardware
+ - Created src/audio/backend/silent_backend.{h,cc}
+ - 7 comprehensive tests: lifecycle, peak control, tracking, playback time, buffer management
+ - Significantly improved audio.cc test coverage
+ - [x] **Backend Reorganization**: Moved all backends to src/audio/backend/ subdirectory
+ - Cleaner code organization, updated all includes and CMake paths
+ - [x] **Dead Code Removal**: Removed unused `register_spec_asset()` function
+ - **Result**: All 28 tests passing, perfect audio-visual sync, improved test coverage
+
## Recently Completed (February 6, 2026)
- [x] **Critical Shader Bug Fixes & Test Infrastructure** (February 6, 2026)
diff --git a/tools/spectral_editor/script.js b/tools/spectral_editor/script.js
index 392d3a5..cafd7e9 100644
--- a/tools/spectral_editor/script.js
+++ b/tools/spectral_editor/script.js
@@ -1083,18 +1083,7 @@ function drawReferenceSpectrogram(ctx) {
// Sample spectrogram
const specValue = state.referenceSpectrogram[frameIdx * state.referenceDctSize + bin];
-
- // Logarithmic intensity mapping (dB scale)
- // Maps wide dynamic range to visible range
- const amplitude = Math.abs(specValue);
- let intensity = 0;
- if (amplitude > 0.0001) { // Noise floor
- const dB = 20.0 * Math.log10(amplitude);
- const dB_min = -60.0; // Noise floor (-60 dB)
- const dB_max = 40.0; // Peak (40 dB headroom)
- const normalized = (dB - dB_min) / (dB_max - dB_min);
- intensity = Math.floor(Math.max(0, Math.min(255, normalized * 255)));
- }
+ const intensity = amp_to_dB(specValue, 255);
// Write pixel
const pixelIdx = (screenY * state.canvasWidth + screenX) * 4;
@@ -1153,24 +1142,13 @@ function drawProceduralSpectrogram(ctx) {
const specValue = curveSpec[frameIdx * state.referenceDctSize + bin];
// Logarithmic intensity mapping with steeper falloff for procedural curves
- const amplitude = Math.abs(specValue);
- let intensity = 0.0;
- if (amplitude > 0.001) { // Higher noise floor for cleaner visualization
- const dB = 20.0 * Math.log10(amplitude);
- const dB_min = -40.0; // Higher floor = steeper falloff (was -60)
- const dB_max = 40.0; // Peak
- const normalized = (dB - dB_min) / (dB_max - dB_min);
- intensity = Math.max(0, Math.min(1.0, normalized)); // 0.0 to 1.0
- }
-
- if (intensity > 0.01) { // Only draw visible pixels
- const pixelIdx = (screenY * state.canvasWidth + screenX) * 4;
- // Use constant color with alpha for intensity (pure colors)
- imgData.data[pixelIdx + 0] = color.r;
- imgData.data[pixelIdx + 1] = color.g;
- imgData.data[pixelIdx + 2] = color.b;
- imgData.data[pixelIdx + 3] = Math.floor(intensity * 255); // Alpha = intensity
- }
+ const intensity = amp_to_dB(specValue, 255.);
+ const pixelIdx = (screenY * state.canvasWidth + screenX) * 4;
+ // Use constant color with alpha for intensity (pure colors)
+ imgData.data[pixelIdx + 0] = color.r;
+ imgData.data[pixelIdx + 1] = color.g;
+ imgData.data[pixelIdx + 2] = color.b;
+ imgData.data[pixelIdx + 3] = intensity;
}
}
@@ -1468,6 +1446,19 @@ function updatePlayhead() {
requestAnimationFrame(updatePlayhead);
}
+// Logarithmic intensity mapping (dB scale)
+// Maps wide dynamic range to visible range
+function amp_to_dB(amplitude, max_value) {
+ amplitude = Math.abs(amplitude);
+ if (amplitude < 0.0001) return 0; // noise floor
+ const dB_min = -40.0; // Noise floor
+ const dB_max = 40.0; // Peak (40 dB headroom)
+ const dB_scale = max_value * (1. / (dB_max - dB_min));
+ const dB = 20.0 * Math.log10(amplitude);
+ const normalized = (dB - dB_min) * dB_scale;
+ return Math.floor(Math.max(0, Math.min(max_value, normalized)));
+}
+
function drawSpectrumViewer() {
const viewer = document.getElementById('spectrumViewer');
const canvas = document.getElementById('spectrumCanvas');
@@ -1485,7 +1476,7 @@ function drawSpectrumViewer() {
frameIdx = state.mouseFrame;
}
- if (frameIdx < 0 || frameIdx >= (state.referenceNumFrames || 100)) return;
+ if (frameIdx < 0 || frameIdx >= state.referenceNumFrames) return;
// Clear canvas
ctx.fillStyle = '#1e1e1e';
@@ -1494,17 +1485,37 @@ function drawSpectrumViewer() {
const numBars = 100; // Downsample to 100 bars for performance
const barWidth = canvas.width / numBars;
+ // Draw spectrum bars (both reference and procedural overlaid)
+ function draw_spectrum(spectrum, is_ref_spectrum) {
+ if (!spectrum) return;
+ for (let i = 0; i < numBars; i++) {
+ const binIdx = Math.floor(i * state.referenceDctSize / numBars);
+ const height = amp_to_dB(spectrum[binIdx], canvas.height);
+ const gradient = ctx.createLinearGradient(0, canvas.height - height, 0, canvas.height);
+ if (is_ref_spectrum) {
+ // Draw reference spectrum (green, behind)
+ gradient.addColorStop(0, '#00ff00');
+ gradient.addColorStop(1, '#004400');
+ } else {
+ // Draw procedural spectrum (red, overlaid)
+ gradient.addColorStop(0, '#ff5555'); // Bright red
+ gradient.addColorStop(1, '#550000'); // Dark red
+ ctx.globalAlpha = 0.7;
+ }
+ ctx.fillStyle = gradient;
+ ctx.fillRect(i * barWidth, canvas.height - height, barWidth, height);
+ ctx.globalAlpha = 1.0;
+ }
+ }
+
+ const size = state.referenceDctSize;
+ const pos = frameIdx * size;
// Get reference spectrum (if available)
- let refSpectrum = null;
if (state.referenceSpectrogram && frameIdx < state.referenceNumFrames) {
- refSpectrum = new Float32Array(state.referenceDctSize);
- for (let bin = 0; bin < state.referenceDctSize; bin++) {
- refSpectrum[bin] = state.referenceSpectrogram[frameIdx * state.referenceDctSize + bin];
- }
+ draw_spectrum(state.referenceSpectrogram.subarray(pos, pos + size), true);
}
// Get procedural spectrum (if curves exist)
- let procSpectrum = null;
if (state.curves.length > 0) {
const numFrames = state.referenceNumFrames || 100;
const fullProcSpec = new Float32Array(state.referenceDctSize * numFrames);
@@ -1513,60 +1524,7 @@ function drawSpectrumViewer() {
});
// Extract just this frame
- procSpectrum = new Float32Array(state.referenceDctSize);
- for (let bin = 0; bin < state.referenceDctSize; bin++) {
- procSpectrum[bin] = fullProcSpec[frameIdx * state.referenceDctSize + bin];
- }
- }
-
- // Draw spectrum bars (both reference and procedural overlaid)
- for (let i = 0; i < numBars; i++) {
- const binIdx = Math.floor(i * state.referenceDctSize / numBars);
-
- // Draw reference spectrum (green, behind)
- if (refSpectrum) {
- const amplitude = Math.abs(refSpectrum[binIdx]);
- let height = 0;
- if (amplitude > 0.0001) {
- const dB = 20.0 * Math.log10(amplitude);
- const dB_min = -60.0;
- const dB_max = 40.0;
- const normalized = (dB - dB_min) / (dB_max - dB_min);
- height = Math.max(0, Math.min(canvas.height, normalized * canvas.height));
- }
-
- if (height > 0) {
- const gradient = ctx.createLinearGradient(0, canvas.height - height, 0, canvas.height);
- gradient.addColorStop(0, '#00ff00');
- gradient.addColorStop(1, '#004400');
- ctx.fillStyle = gradient;
- ctx.fillRect(i * barWidth, canvas.height - height, barWidth - 1, height);
- }
- }
-
- // Draw procedural spectrum (red, overlaid)
- if (procSpectrum) {
- const amplitude = Math.abs(procSpectrum[binIdx]);
- let height = 0;
- if (amplitude > 0.001) {
- const dB = 20.0 * Math.log10(amplitude);
- const dB_min = -40.0; // Same as procedural spectrogram rendering
- const dB_max = 40.0;
- const normalized = (dB - dB_min) / (dB_max - dB_min);
- height = Math.max(0, Math.min(canvas.height, normalized * canvas.height));
- }
-
- if (height > 0) {
- const gradient = ctx.createLinearGradient(0, canvas.height - height, 0, canvas.height);
- gradient.addColorStop(0, '#ff5555'); // Bright red
- gradient.addColorStop(1, '#550000'); // Dark red
- ctx.fillStyle = gradient;
- // Make it slightly transparent to see overlap
- ctx.globalAlpha = 0.7;
- ctx.fillRect(i * barWidth, canvas.height - height, barWidth - 1, height);
- ctx.globalAlpha = 1.0;
- }
- }
+ draw_spectrum(fullProcSpec.subarray(pos, pos + size), false);
}
// Draw frequency labels