diff options
| -rw-r--r-- | doc/AUDIO_LIFECYCLE_REFACTOR.md | 388 |
1 files changed, 366 insertions, 22 deletions
diff --git a/doc/AUDIO_LIFECYCLE_REFACTOR.md b/doc/AUDIO_LIFECYCLE_REFACTOR.md index adf0045..1d73bb1 100644 --- a/doc/AUDIO_LIFECYCLE_REFACTOR.md +++ b/doc/AUDIO_LIFECYCLE_REFACTOR.md @@ -294,13 +294,17 @@ class SpectrogramResourceManager { void init(); void shutdown(); // Frees all owned memory - // Loading API - int load_asset_spectrogram(AssetId asset_id); - int generate_procedural_spectrogram(const NoteParams& params); + // Lazy loading API (loads on first access) + const Spectrogram* get_or_load_asset(AssetId asset_id); + const Spectrogram* get_or_generate_procedural(int sample_id, const NoteParams& params); + + // Explicit loading (for pre-warming if needed) + int preload_asset(AssetId asset_id); + int preload_procedural(int sample_id, const NoteParams& params); // Query API const Spectrogram* get_spectrogram(int resource_id) const; - bool is_valid(int resource_id) const; + bool is_loaded(int resource_id) const; // Cache management void clear_cache(); @@ -319,7 +323,7 @@ class SpectrogramResourceManager { }; ``` -### Usage in AudioEngine +### Usage in AudioEngine (Lazy Loading) ```cpp class AudioEngine { @@ -334,34 +338,65 @@ class AudioEngine { const NoteParams* samples, const AssetId* sample_assets, uint32_t sample_count) { - // Load all resources ONCE at init + // Register sample metadata (but don't load yet!) for (uint32_t i = 0; i < sample_count; ++i) { - int resource_id; if (sample_assets[i] != AssetId::ASSET_LAST_ID) { - resource_id = resource_mgr_.load_asset_spectrogram(sample_assets[i]); + resource_mgr_.register_asset(i, sample_assets[i]); // Metadata only } else { - resource_id = resource_mgr_.generate_procedural_spectrogram(samples[i]); + resource_mgr_.register_procedural(i, samples[i]); // Metadata only } + } - // Register with synth - const Spectrogram* spec = resource_mgr_.get_spectrogram(resource_id); - int synth_id = synth_.register_spectrogram(spec); + tracker_.load_score(score); + } - // Store mapping for tracker - sample_to_synth_id_[i] = synth_id; - } + void update(float music_time, float dt) { + // Pre-warm samples needed in next 2 seconds + tracker_.prewarm_lookahead(music_time, music_time + 2.0f); - tracker_.load_score(score, sample_to_synth_id_); + // Update tracker (triggers events) + tracker_.update(music_time); + } + + void trigger_sample(int sample_id, float volume, float pan) { + // Load on-demand if not cached + const Spectrogram* spec = resource_mgr_.get_or_load(sample_id); + + // Register with synth (lazy registration) + int synth_id = get_or_register_synth_id(sample_id, spec); + + // Trigger voice + synth_.trigger_voice(synth_id, volume, pan); } private: + int get_or_register_synth_id(int sample_id, const Spectrogram* spec) { + if (sample_to_synth_id_[sample_id] == -1) { + // First use: register with synth + sample_to_synth_id_[sample_id] = synth_.register_spectrogram(spec); + } + return sample_to_synth_id_[sample_id]; + } + Synth synth_; Tracker tracker_; SpectrogramResourceManager resource_mgr_; - int sample_to_synth_id_[256]; + int sample_to_synth_id_[256] = {-1}; // -1 = not registered yet }; ``` +**Timeline:** +``` +t=0.0s: init() - No samples loaded (instant startup) +t=0.0s: load_music_data() - Register metadata only (~0ms) +t=0.0s: update(0.0) - Pre-warm samples for t=0-2s (load 3-5 samples) +t=0.0s: Pattern triggers → trigger_sample() → Instant (pre-warmed) +t=1.0s: update(1.0) - Pre-warm samples for t=1-3s (load 2-3 more) +t=2.0s: update(2.0) - Pre-warm samples for t=2-4s (load 2-3 more) +... +Total samples loaded by t=10s: Maybe 10-12 (not all 19) +``` + ### Memory Ownership Rules **Clear Ownership:** @@ -374,6 +409,216 @@ class AudioEngine { - Procedurals live until `ResourceManager::shutdown()` - Synth references valid as long as ResourceManager exists +--- + +## Lazy Loading & On-Demand Strategy + +### Problem with Eager Loading + +**Current plan says:** "Load all resources ONCE at init" + +**Issues:** +- Long startup delay (load all 19 samples before demo starts) +- Wasted memory (samples used only once stay loaded) +- No benefit for streaming or late-game content + +**Example:** Demo is 60s, sample only used at t=45s → why load it at t=0? + +### Proposed: Lazy Loading with Cache + +**Strategy:** +1. **Load on first trigger** - Don't load until sample is actually needed +2. **Cache aggressively** - Keep loaded samples in memory (demo is short, ~60s) +3. **Optional pre-warming** - Load predicted samples 1-2 seconds ahead +4. **Eviction policy** - Unload samples that won't be used again (optional) + +### Implementation: Two-Tier Cache + +```cpp +class SpectrogramResourceManager { + private: + struct CacheEntry { + Spectrogram spec; + float* owned_data; // nullptr if not loaded yet + AssetId asset_id; + NoteParams proc_params; // For procedurals + + enum State { + UNLOADED, // Not loaded yet + LOADING, // Load in progress (async) + LOADED, // Ready to use + EVICTED // Was loaded, now evicted + }; + State state = UNLOADED; + float last_access_time; // For LRU eviction + }; + + CacheEntry cache_[MAX_SPECTROGRAMS]; + + public: + // Lazy loading (loads if not cached) + const Spectrogram* get_or_load(int sample_id) { + CacheEntry& entry = cache_[sample_id]; + + if (entry.state == LOADED) { + entry.last_access_time = current_time(); + return &entry.spec; // Cache hit! + } + + if (entry.state == UNLOADED || entry.state == EVICTED) { + load_now(&entry); // Load synchronously + entry.state = LOADED; + } + + return &entry.spec; + } + + private: + void load_now(CacheEntry* entry) { + if (entry->asset_id != ASSET_LAST_ID) { + // ASSET: Borrow pointer from AssetManager + size_t size; + const uint8_t* data = GetAsset(entry->asset_id, &size); + + #if defined(SUPPORT_COMPRESSED_ASSETS) + // ON-DEMAND DECOMPRESSION + if (is_compressed(data)) { + entry->owned_data = decompress_asset(data, size); + entry->spec.spectral_data_a = entry->owned_data; + } else { + entry->spec.spectral_data_a = (const float*)(data + sizeof(SpecHeader)); + } + #else + // Direct pointer (no decompression) + entry->spec.spectral_data_a = (const float*)(data + sizeof(SpecHeader)); + #endif + + } else { + // PROCEDURAL: Generate on-demand + int frames; + std::vector<float> data = generate_note_spectrogram(entry->proc_params, &frames); + + entry->owned_data = new float[data.size()]; + memcpy(entry->owned_data, data.data(), data.size() * sizeof(float)); + entry->spec.spectral_data_a = entry->owned_data; + entry->spec.num_frames = frames; + } + } +}; +``` + +### On-Demand Decompression (Future) + +When assets are compressed (Task #27), decompress lazily: + +```cpp +const Spectrogram* get_or_load(int sample_id) { + CacheEntry& entry = cache_[sample_id]; + + if (entry.state == LOADED) { + return &entry.spec; // Already decompressed + } + + // Load compressed asset + const uint8_t* compressed = GetAsset(entry->asset_id, &size); + + // Decompress into owned buffer + entry->owned_data = decompress_zlib(compressed, size, &decompressed_size); + entry->spec.spectral_data_a = entry->owned_data; + entry->state = LOADED; + + return &entry.spec; +} + +void evict(int sample_id) { + // Free decompressed data, can re-decompress from asset later + delete[] cache_[sample_id].owned_data; + cache_[sample_id].owned_data = nullptr; + cache_[sample_id].state = EVICTED; +} +``` + +### Pre-warming Strategy + +**Idea:** Predict samples needed in next 1-2 seconds, load ahead of time + +```cpp +class Tracker { + public: + void update(float music_time) { + // 1. Trigger patterns at current time + trigger_patterns_at(music_time); + + // 2. Pre-warm samples needed in next 2 seconds + prewarm_lookahead(music_time, music_time + 2.0f); + } + + private: + void prewarm_lookahead(float start_time, float end_time) { + // Scan upcoming pattern triggers + for (const auto& trigger : upcoming_triggers_) { + if (trigger.time > start_time && trigger.time < end_time) { + const TrackerPattern& pattern = get_pattern(trigger.pattern_id); + + // Load all samples used in this pattern + for (const auto& event : pattern.events) { + resource_mgr_->preload(event.sample_id); // Async or sync + } + } + } + } +}; +``` + +**Benefits:** +- No loading stutter when pattern triggers +- Spreads load cost over time (not all at init) +- Can load asynchronously in background thread (if needed) + +### Cache Eviction Policy (Optional) + +**For very long demos (>5 minutes) or memory-constrained systems:** + +```cpp +void try_evict_lru() { + if (get_cache_memory_usage() > MAX_CACHE_SIZE) { + // Find least recently used entry + int lru_id = find_lru_entry(); + + // Only evict if not used recently + if (current_time() - cache_[lru_id].last_access_time > 10.0f) { + evict(lru_id); + } + } +} +``` + +**Eviction strategies:** +1. **LRU** (Least Recently Used) - evict oldest access +2. **Time-based** - evict samples >10s since last use +3. **Timeline hints** - demo tells cache "won't need sample X again" + +**For 64k demo:** Probably not needed (short duration, small sample count) + +--- + +### Comparison: Eager vs Lazy Loading + +| Aspect | Eager (Load All at Init) | Lazy (Load on Demand) | Lazy + Pre-warm | +|--------|-------------------------|----------------------|-----------------| +| **Startup Time** | ❌ Slow (load all 19) | ✅ Fast (load 0) | ✅ Fast (load 0) | +| **First Trigger Latency** | ✅ Instant (cached) | ❌ Stutter (load now) | ✅ Instant (pre-loaded) | +| **Memory Usage** | ❌ High (all loaded) | ✅ Low (only used) | ⚠️ Medium (active + lookahead) | +| **Complexity** | ✅ Simple | ⚠️ Medium | ❌ Complex | +| **Best For** | Short demos, few samples | Long demos, many samples | **Recommended for 64k** | + +**Recommendation:** **Lazy + Pre-warm (1-2s lookahead)** +- Fast startup (no eager loading) +- No stutter (pre-warming prevents load-on-trigger) +- Memory efficient (only loads active + upcoming samples) + +--- + ### Benefits Over Current Design | Aspect | Current (Tracker-owned) | Proposed (ResourceManager) | @@ -435,9 +680,12 @@ class AssetManagerAdapter : public IAssetProvider { ### Phase 1: Design & Prototype (5-7 days) - [ ] Create `src/audio/spectrogram_resource_manager.h/cc` + - [ ] Implement lazy loading cache (UNLOADED → LOADED states) + - [ ] Add `get_or_load()` API for on-demand loading + - [ ] Add `register_*()` API for metadata-only registration - [ ] Implement resource loading (assets + procedurals) - [ ] Add memory ownership tracking - - [ ] Write unit tests for resource lifecycle + - [ ] Write unit tests for lazy loading behavior - [ ] Create `src/audio/audio_engine.h/cc` with class interface - [ ] Implement `AudioEngine::init()` / `shutdown()` / `reset()` - [ ] Add delegation methods for synth/tracker APIs @@ -471,6 +719,72 @@ class AssetManagerAdapter : public IAssetProvider { - [ ] Profile binary size impact - [ ] Optimize if >1KB overhead - [ ] Consider `#if !defined(STRIP_ALL)` for test-only features +- [ ] **Optional:** Add async loading (background thread for pre-warming) + - Load samples in background while demo runs + - Requires mutex for cache access + - Benefit: Zero stutter even for large samples + - Cost: +200-300 bytes for threading code + +--- + +## Future: Async Loading (Post-MVP) + +**Idea:** Pre-warm samples in background thread to avoid any stutter + +```cpp +class SpectrogramResourceManager { + public: + void prewarm_async(int sample_id) { + if (cache_[sample_id].state == UNLOADED) { + cache_[sample_id].state = LOADING; + + // Enqueue background load + thread_pool_.enqueue([this, sample_id]() { + load_now(&cache_[sample_id]); + + std::lock_guard lock(cache_mutex_); + cache_[sample_id].state = LOADED; + }); + } + } + + const Spectrogram* get_or_load(int sample_id) { + std::lock_guard lock(cache_mutex_); + + CacheEntry& entry = cache_[sample_id]; + + if (entry.state == LOADED) { + return &entry.spec; // Ready! + } + + if (entry.state == LOADING) { + // Wait for background load to finish + wait_for_load(sample_id); + } else { + // Not started, load synchronously (fallback) + load_now(&entry); + entry.state = LOADED; + } + + return &entry.spec; + } + + private: + std::mutex cache_mutex_; + ThreadPool thread_pool_; +}; +``` + +**Benefits:** +- Zero latency: All loads happen in background +- Demo runs at full framerate during loading + +**Costs:** +- +200-300 bytes for threading code +- Complexity: Need mutexes, thread management +- May not be needed for 64k (samples are small, load quickly) + +**Recommendation:** Only implement if profiling shows stutter issues --- @@ -552,15 +866,45 @@ class AudioEngine { - Cleared automatically on `engine.reset()` - Testable (can verify cache contents) +### Q: Why not load all samples at init? + +**A:** Lazy loading with pre-warming is more efficient: + +**Problems with eager loading:** +- Slow startup: Load all 19 samples before t=0 (~50-100ms delay) +- Wasted memory: Sample used only at t=45s stays loaded from t=0 +- No benefit: 60s demo doesn't need all samples loaded upfront + +**Lazy + Pre-warm strategy:** +``` +t=0.0s: Load 0 samples (instant startup) +t=0.0s: Pre-warm 3-5 samples for t=0-2s range (background) +t=1.0s: Pre-warm 2-3 more for t=1-3s range +By t=10s: Only ~10 samples loaded (not all 19) +``` + +**Benefits:** +- ✅ Instant startup (no initial loading) +- ✅ No trigger stutter (pre-warming prevents load-on-access) +- ✅ Memory efficient (only active + upcoming samples) +- ✅ Spreads load cost over time + +**Future: On-demand decompression (Task #27)** +When assets are compressed, decompress only when needed: +- Compressed asset in binary: 2KB +- Decompressed in memory: 8KB (4x larger) +- Only decompress samples actually used (save memory) + ### Q: How does this affect binary size? **Estimated overhead:** -- `SpectrogramResourceManager`: ~300 bytes (struct + logic) -- `AudioEngine` wrapper: ~200 bytes (vtable + members) -- Total: **~500 bytes** (0.8% of 64KB budget) +- `SpectrogramResourceManager` (basic): ~300 bytes +- `SpectrogramResourceManager` (with lazy loading): ~500 bytes +- `AudioEngine` wrapper: ~200 bytes +- Total: **~700 bytes** (1.1% of 64KB budget) **Savings from removing global state:** ~100 bytes -**Net impact:** **~400 bytes** (acceptable for robustness) +**Net impact:** **~600 bytes** (acceptable for robustness + lazy loading) --- |