diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/app/main.cc | 43 | ||||
| -rw-r--r-- | src/app/test_demo.cc | 4 | ||||
| -rw-r--r-- | src/audio/audio.cc | 34 | ||||
| -rw-r--r-- | src/audio/tracker.cc | 19 | ||||
| -rw-r--r-- | src/effects/cnn_effect.cc | 129 | ||||
| -rw-r--r-- | src/effects/cnn_effect.h | 53 | ||||
| -rw-r--r-- | src/effects/cnn_v2_effect.cc | 497 | ||||
| -rw-r--r-- | src/effects/cnn_v2_effect.h | 89 | ||||
| -rw-r--r-- | src/effects/flash_cube_effect.cc | 6 | ||||
| -rw-r--r-- | src/effects/gaussian_blur_effect.h | 4 | ||||
| -rw-r--r-- | src/effects/particle_spray_effect.h | 2 | ||||
| -rw-r--r-- | src/effects/particles_effect.h | 2 | ||||
| -rw-r--r-- | src/effects/sdf_test_effect.cc | 4 | ||||
| -rw-r--r-- | src/gpu/demo_effects.h | 8 | ||||
| -rw-r--r-- | src/gpu/gpu.cc | 1 | ||||
| -rw-r--r-- | src/tests/audio/test_audio_engine.cc | 8 | ||||
| -rw-r--r-- | src/tests/gpu/test_demo_effects.cc | 4 | ||||
| -rw-r--r-- | src/tests/gpu/test_shader_assets.cc | 4 |
18 files changed, 59 insertions, 852 deletions
diff --git a/src/app/main.cc b/src/app/main.cc index 537da74..3c80520 100644 --- a/src/app/main.cc +++ b/src/app/main.cc @@ -207,7 +207,10 @@ int main(int argc, char** argv) { #endif /* !defined(STRIP_ALL) */ // Pre-fill ring buffer to target lookahead (prevents startup delay) - fill_audio_buffer(audio_get_required_prefill_time(), 0.0); + // Skip pre-fill in WAV dump mode (direct render, no ring buffer) + if (!dump_wav) { + fill_audio_buffer(audio_get_required_prefill_time(), 0.0); + } audio_start(); g_last_audio_time = audio_get_playback_time(); // Initialize after start @@ -268,37 +271,41 @@ int main(int argc, char** argv) { printf("Running WAV dump simulation (%.1fs - %.1fs)...\n", start_time, end_time); - // Seek to start time if needed + // Seek to start time if needed (advance state without rendering) if (start_time > 0.0f) { const double step = 1.0 / 60.0; for (double t = 0.0; t < start_time; t += step) { - fill_audio_buffer(step, t); - audio_render_silent((float)step); + g_audio_engine.update(g_music_time, (float)step * g_tempo_scale); + g_music_time += (float)step * g_tempo_scale; } printf("Seeked to %.1fs\n", start_time); } - const float update_dt = 1.0f / 60.0f; // 60Hz update rate - const int frames_per_update = (int)(32000 * update_dt); // ~533 frames - const int samples_per_update = frames_per_update * 2; // Stereo + const float update_dt = 1.0f / 60.0f; // 60Hz update rate + const int sample_rate = 32000; - AudioRingBuffer* ring_buffer = audio_get_ring_buffer(); - std::vector<float> chunk_buffer(samples_per_update); + std::vector<float> chunk_buffer(2048); // Max samples for one update double physical_time = start_time; + double frame_accumulator = 0.0; while (physical_time < end_time) { - // Update music time and tracker (using tempo logic from - // fill_audio_buffer) - fill_audio_buffer(update_dt, physical_time); + // Calculate exact frames for this update + frame_accumulator += sample_rate * update_dt; + const int frames_this_update = (int)frame_accumulator; + frame_accumulator -= frames_this_update; + const int samples_this_update = frames_this_update * 2; - // Read rendered audio from ring buffer - if (ring_buffer != nullptr) { - ring_buffer->read(chunk_buffer.data(), samples_per_update); - } + // Update tracker/audio state + g_audio_engine.update(g_music_time, update_dt * g_tempo_scale); - // Write to WAV file - wav_backend.write_audio(chunk_buffer.data(), samples_per_update); + // Render directly to buffer (bypass ring buffer) + if (frames_this_update > 0) { + synth_render(chunk_buffer.data(), frames_this_update); + wav_backend.write_audio(chunk_buffer.data(), samples_this_update); + } + // Advance music time + g_music_time += update_dt * g_tempo_scale; physical_time += update_dt; // Progress indicator every second diff --git a/src/app/test_demo.cc b/src/app/test_demo.cc index 5775e74..ff2c105 100644 --- a/src/app/test_demo.cc +++ b/src/app/test_demo.cc @@ -20,8 +20,8 @@ extern float GetDemoDuration(); extern void LoadTimeline(MainSequence& main_seq, const GpuContext& ctx); // Inline peak meter effect for debugging audio-visual sync -#include "effects/cnn_effect.h" -#include "effects/cnn_v2_effect.h" +#include "../../cnn_v1/src/cnn_v1_effect.h" +#include "../../cnn_v2/src/cnn_v2_effect.h" #include "gpu/post_process_helper.h" #include "gpu/shader_composer.h" diff --git a/src/audio/audio.cc b/src/audio/audio.cc index f5bc4ab..a220fbb 100644 --- a/src/audio/audio.cc +++ b/src/audio/audio.cc @@ -78,9 +78,9 @@ void audio_start() { #if !defined(STRIP_ALL) if (!audio_is_prefilled()) { const int buffered = g_ring_buffer.available_read(); - const float buffered_ms = - (float)buffered / (RING_BUFFER_SAMPLE_RATE * RING_BUFFER_CHANNELS) * - 1000.0f; + const float buffered_ms = (float)buffered / + (RING_BUFFER_SAMPLE_RATE * RING_BUFFER_CHANNELS) * + 1000.0f; printf("WARNING: Audio buffer not pre-filled (%.1fms < %.1fms)\n", buffered_ms, audio_get_required_prefill_time() * 1000.0f); } @@ -97,21 +97,18 @@ void audio_render_ahead(float music_time, float dt, float target_fill) { // Render in small chunks to keep synth time synchronized with tracker // Chunk size: one frame's worth of audio (~16.6ms @ 60fps) - // TODO(timing): CRITICAL BUG - Truncation here may cause 180ms drift over 63 beats - // (int) cast loses fractional samples: 0.333 samples/frame * 2560 frames = 853 samples = 27ms - // But observed drift is 180ms, so this is not the only source (27ms < 180ms) - // NOTE: This is NOT a float vs double precision issue - floats handle <500s times fine - // See also: tracker.cc BPM timing calculation + // TODO(timing): CRITICAL BUG - Truncation here may cause 180ms drift over 63 + // beats (int) cast loses fractional samples: 0.333 samples/frame * 2560 + // frames = 853 samples = 27ms But observed drift is 180ms, so this is not the + // only source (27ms < 180ms) NOTE: This is NOT a float vs double precision + // issue - floats handle <500s times fine See also: tracker.cc BPM timing + // calculation const int chunk_frames = (int)(dt * RING_BUFFER_SAMPLE_RATE); const int chunk_samples = chunk_frames * RING_BUFFER_CHANNELS; if (chunk_frames <= 0) return; - static int64_t g_total_render_calls = 0; - static int64_t g_total_frames_rendered = 0; - const int64_t frames_before = g_ring_buffer.get_total_written() / RING_BUFFER_CHANNELS; - // Keep rendering small chunks until buffer is full enough while (true) { // First, try to flush any pending samples from previous partial writes @@ -228,19 +225,6 @@ void audio_render_ahead(float music_time, float dt, float target_fill) { } } } - - // DEBUG: Track actual frames rendered vs expected - const int64_t frames_after = g_ring_buffer.get_total_written() / RING_BUFFER_CHANNELS; - const int64_t actual_rendered = frames_after - frames_before; - g_total_render_calls++; - g_total_frames_rendered += actual_rendered; - - if (g_total_render_calls % 600 == 0) { // Every 10 seconds at 60fps - const float expected_frames = g_total_render_calls * (float)(chunk_frames); - const float drift_ms = (expected_frames - g_total_frames_rendered) / RING_BUFFER_SAMPLE_RATE * 1000.0f; - printf("[RENDER_DRIFT] calls=%lld expect=%.1f actual=%lld drift=%.2fms\n", - g_total_render_calls, expected_frames, g_total_frames_rendered, drift_ms); - } } float audio_get_playback_time() { diff --git a/src/audio/tracker.cc b/src/audio/tracker.cc index 00c31e9..38c814d 100644 --- a/src/audio/tracker.cc +++ b/src/audio/tracker.cc @@ -193,7 +193,8 @@ static int get_free_pattern_slot() { // sample-accurate timing) // volume_mult: Additional volume multiplier (for humanization) static void trigger_note_event(const TrackerEvent& event, - int start_offset_samples, float volume_mult = 1.0f) { + int start_offset_samples, + float volume_mult = 1.0f) { #if defined(DEBUG_LOG_TRACKER) // VALIDATION: Check sample_id bounds if (event.sample_id >= g_tracker_samples_count) { @@ -234,10 +235,10 @@ static void trigger_note_event(const TrackerEvent& event, } void tracker_update(float music_time_sec, float dt_music_sec) { - // TODO(timing): CRITICAL BUG - Events trigger ~180ms early over 63 beats @ BPM=90 - // Observed: Beat 63 snare at 41.82s in WAV, should be at 42.00s (180ms drift) - // NOTE: This is NOT a float vs double precision issue - floats handle <500s times fine - // Root cause unknown - suspects: + // TODO(timing): CRITICAL BUG - Events trigger ~180ms early over 63 beats @ + // BPM=90 Observed: Beat 63 snare at 41.82s in WAV, should be at 42.00s (180ms + // drift) NOTE: This is NOT a float vs double precision issue - floats handle + // <500s times fine Root cause unknown - suspects: // 1. Systematic bias in time calculation (not random accumulation) // 2. Truncation in audio.cc:103 chunk_frames = (int)(dt * sample_rate) // 3. BPM calculation precision below (unit_duration_sec) @@ -324,14 +325,6 @@ void tracker_update(float music_time_sec, float dt_music_sec) { } } - // DEBUG: Track kick/snare timing for drift investigation - if (event.sample_id == 0 || event.sample_id == 1) { // Assuming kick=0, snare=1 - const char* name = (event.sample_id == 0) ? "KICK " : "SNARE"; - const float delta_ms = (event_music_time - music_time_sec) * 1000.0f; - printf("[DRIFT] %s: music=%.4f expect=%.4f delta=%.2fms offset=%d\n", - name, music_time_sec, event_music_time, delta_ms, sample_offset); - } - trigger_note_event(event, sample_offset, volume_mult); active.next_event_idx++; } diff --git a/src/effects/cnn_effect.cc b/src/effects/cnn_effect.cc deleted file mode 100644 index 49c5239..0000000 --- a/src/effects/cnn_effect.cc +++ /dev/null @@ -1,129 +0,0 @@ -// CNN post-processing effect implementation -// Neural network-based stylization with modular WGSL - -#include "effects/cnn_effect.h" -#include "gpu/bind_group_builder.h" -#include "gpu/effect.h" -#include "gpu/pipeline_builder.h" -#include "gpu/post_process_helper.h" -#include "gpu/sampler_cache.h" -#include "gpu/shader_composer.h" -#include "gpu/shaders.h" - -// Create custom pipeline with 5 bindings (includes original texture) -static WGPURenderPipeline create_cnn_pipeline(WGPUDevice device, - WGPUTextureFormat format, - const char* shader_code) { - WGPUBindGroupLayout bgl = - BindGroupLayoutBuilder() - .sampler(0, WGPUShaderStage_Fragment) - .texture(1, WGPUShaderStage_Fragment) - .uniform(2, WGPUShaderStage_Vertex | WGPUShaderStage_Fragment) - .uniform(3, WGPUShaderStage_Fragment) - .texture(4, WGPUShaderStage_Fragment) - .build(device); - - WGPURenderPipeline pipeline = RenderPipelineBuilder(device) - .shader(shader_code) - .bind_group_layout(bgl) - .format(format) - .build(); - - wgpuBindGroupLayoutRelease(bgl); - return pipeline; -} - -CNNEffect::CNNEffect(const GpuContext& ctx) - : PostProcessEffect(ctx), layer_index_(0), total_layers_(1), - blend_amount_(1.0f), input_view_(nullptr), original_view_(nullptr), - bind_group_(nullptr) { - pipeline_ = - create_cnn_pipeline(ctx_.device, ctx_.format, cnn_layer_shader_wgsl); -} - -CNNEffect::CNNEffect(const GpuContext& ctx, const CNNEffectParams& params) - : PostProcessEffect(ctx), layer_index_(params.layer_index), - total_layers_(params.total_layers), blend_amount_(params.blend_amount), - input_view_(nullptr), original_view_(nullptr), bind_group_(nullptr) { - pipeline_ = - create_cnn_pipeline(ctx_.device, ctx_.format, cnn_layer_shader_wgsl); -} - -void CNNEffect::init(MainSequence* demo) { - PostProcessEffect::init(demo); - demo_ = demo; - params_buffer_.init(ctx_.device); - - // Register auxiliary texture for layer 0 (width_/height_ set by resize()) - if (layer_index_ == 0) { - demo_->register_auxiliary_texture("captured_frame", width_, height_); - } - - // Initialize uniforms BEFORE any bind group creation - uniforms_.update(ctx_.queue, get_common_uniforms()); - - CNNLayerParams params = {layer_index_, blend_amount_, {0.0f, 0.0f}}; - params_buffer_.update(ctx_.queue, params); -} - -void CNNEffect::resize(int width, int height) { - if (width == width_ && height == height_) - return; - - PostProcessEffect::resize(width, height); - - // Only layer 0 owns the captured_frame texture - if (layer_index_ == 0 && demo_) { - demo_->resize_auxiliary_texture("captured_frame", width, height); - } -} - -void CNNEffect::render(WGPURenderPassEncoder pass, - const CommonPostProcessUniforms& uniforms) { - if (!bind_group_) { - fprintf(stderr, "CNN render: no bind_group\n"); - return; - } - - float effective_blend = blend_amount_; - if (beat_modulated_) { - effective_blend = blend_amount_ * uniforms.beat_phase * beat_scale_; - } - - CNNLayerParams params = {layer_index_, effective_blend, {0.0f, 0.0f}}; - params_buffer_.update(ctx_.queue, params); - - wgpuRenderPassEncoderSetPipeline(pass, pipeline_); - wgpuRenderPassEncoderSetBindGroup(pass, 0, bind_group_, 0, nullptr); - wgpuRenderPassEncoderDraw(pass, 3, 1, 0, 0); -} - -void CNNEffect::update_bind_group(WGPUTextureView input_view) { - input_view_ = input_view; - - // Update common uniforms (CRITICAL for UV calculation!) - uniforms_.update(ctx_.queue, get_common_uniforms()); - - // All layers: get captured frame (original input from layer 0) - if (demo_) { - original_view_ = demo_->get_auxiliary_view("captured_frame"); - } - - // Create bind group with original texture - if (bind_group_) - wgpuBindGroupRelease(bind_group_); - - WGPUBindGroupLayout bgl = wgpuRenderPipelineGetBindGroupLayout(pipeline_, 0); - // Use clamp (not repeat) to match PyTorch Conv2d zero-padding behavior - WGPUSampler sampler = - SamplerCache::Get().get_or_create(ctx_.device, SamplerCache::clamp()); - - bind_group_ = - BindGroupBuilder() - .sampler(0, sampler) - .texture(1, input_view_) - .buffer(2, uniforms_.get().buffer, uniforms_.get().size) - .buffer(3, params_buffer_.get().buffer, params_buffer_.get().size) - .texture(4, original_view_ ? original_view_ : input_view_) - .build(ctx_.device, bgl); -} diff --git a/src/effects/cnn_effect.h b/src/effects/cnn_effect.h deleted file mode 100644 index cdcd656..0000000 --- a/src/effects/cnn_effect.h +++ /dev/null @@ -1,53 +0,0 @@ -// CNN post-processing effect header -// Multi-layer neural network stylization - -#pragma once -#include "gpu/effect.h" -#include "gpu/uniform_helper.h" - -struct CNNLayerParams { - int layer_index; - float blend_amount; // Blend: mix(input, output, blend_amount) - float _pad[2]; -}; -static_assert(sizeof(CNNLayerParams) == 16); - -struct CNNEffectParams { - int layer_index = 0; // Which layer to render (0-based) - int total_layers = 1; // Total number of layers in the CNN - float blend_amount = 1.0f; // Final blend with original input -}; - -class CNNEffect : public PostProcessEffect { - public: - explicit CNNEffect(const GpuContext& ctx); - explicit CNNEffect(const GpuContext& ctx, const CNNEffectParams& params); - - void init(MainSequence* demo) override; - void resize(int width, int height) override; - void render(WGPURenderPassEncoder pass, - const CommonPostProcessUniforms& uniforms) override; - void update_bind_group(WGPUTextureView input_view) override; - - // Layer 0 needs framebuffer capture for original input - bool needs_framebuffer_capture() const override { - return layer_index_ == 0; - } - - void set_beat_modulation(bool enabled, float scale = 1.0f) { - beat_modulated_ = enabled; - beat_scale_ = scale; - } - - private: - int layer_index_; - int total_layers_; - float blend_amount_; - bool beat_modulated_ = false; - float beat_scale_ = 1.0f; - WGPUTextureView input_view_; - WGPUTextureView original_view_; - UniformBuffer<CNNLayerParams> params_buffer_; - WGPUBindGroup bind_group_; - MainSequence* demo_ = nullptr; -}; diff --git a/src/effects/cnn_v2_effect.cc b/src/effects/cnn_v2_effect.cc deleted file mode 100644 index 7127aae..0000000 --- a/src/effects/cnn_v2_effect.cc +++ /dev/null @@ -1,497 +0,0 @@ -// CNN v2 Effect Implementation - -#include "effects/cnn_v2_effect.h" - -#if defined(USE_TEST_ASSETS) -#include "test_assets.h" -#else -#include "generated/assets.h" -#endif - -#include "gpu/bind_group_builder.h" -#include "gpu/gpu.h" -#include "util/asset_manager.h" -#include "util/fatal_error.h" -#include <cstring> - -CNNv2Effect::CNNv2Effect(const GpuContext& ctx) - : PostProcessEffect(ctx), static_pipeline_(nullptr), - static_bind_group_(nullptr), static_params_buffer_(nullptr), - static_features_tex_(nullptr), static_features_view_(nullptr), - linear_sampler_(nullptr), layer_pipeline_(nullptr), - weights_buffer_(nullptr), input_mip_tex_(nullptr), - current_input_view_(nullptr), blend_amount_(1.0f), mip_level_(0), - initialized_(false) { - std::memset(input_mip_view_, 0, sizeof(input_mip_view_)); -} - -CNNv2Effect::CNNv2Effect(const GpuContext& ctx, const CNNv2EffectParams& params) - : PostProcessEffect(ctx), static_pipeline_(nullptr), - static_bind_group_(nullptr), static_params_buffer_(nullptr), - static_features_tex_(nullptr), static_features_view_(nullptr), - linear_sampler_(nullptr), layer_pipeline_(nullptr), - weights_buffer_(nullptr), input_mip_tex_(nullptr), - current_input_view_(nullptr), blend_amount_(params.blend_amount), - mip_level_(0), initialized_(false) { - std::memset(input_mip_view_, 0, sizeof(input_mip_view_)); -} - -CNNv2Effect::~CNNv2Effect() { - cleanup(); -} - -void CNNv2Effect::init(MainSequence* demo) { - (void)demo; - if (initialized_) - return; - - load_weights(); - create_textures(); - create_pipelines(); - - initialized_ = true; -} - -void CNNv2Effect::resize(int width, int height) { - PostProcessEffect::resize(width, height); - cleanup(); - create_textures(); - create_pipelines(); -} - -void CNNv2Effect::load_weights() { - // Load binary weights asset - size_t weights_size = 0; - const uint8_t* weights_data = - (const uint8_t*)GetAsset(AssetId::ASSET_WEIGHTS_CNN_V2, &weights_size); - - if (!weights_data || weights_size < 20) { - // Weights not available - effect will skip - return; - } - - // Parse header - const uint32_t* header = (const uint32_t*)weights_data; - uint32_t magic = header[0]; - uint32_t version = header[1]; - uint32_t num_layers = header[2]; - uint32_t total_weights = header[3]; - - FATAL_CHECK(magic != 0x324e4e43, "Invalid CNN v2 weights magic\n"); // 'CNN2' - - // Support both version 1 (16-byte header) and version 2 (20-byte header with - // mip_level) - // TODO: Version 3 should include feature descriptor for arbitrary - // layout/ordering - if (version == 1) { - mip_level_ = 0; // Default for v1 - } else if (version == 2) { - mip_level_ = header[4]; - } else { - FATAL_ERROR("Unsupported CNN v2 weights version: %u\n", version); - } - - // Parse layer info (20 bytes per layer) - // Offset depends on version: v1=16 bytes (4 u32), v2=20 bytes (5 u32) - const uint32_t header_u32_count = (version == 1) ? 4 : 5; - const uint32_t* layer_data = header + header_u32_count; - for (uint32_t i = 0; i < num_layers; ++i) { - LayerInfo info; - info.kernel_size = layer_data[i * 5 + 0]; - info.in_channels = layer_data[i * 5 + 1]; - info.out_channels = layer_data[i * 5 + 2]; - info.weight_offset = layer_data[i * 5 + 3]; - info.weight_count = layer_data[i * 5 + 4]; - layer_info_.push_back(info); - } - - // Create GPU storage buffer for weights (skip header + layer info, upload - // only weights) - size_t header_size = 20; // 5 u32 - size_t layer_info_size = 20 * num_layers; // 5 u32 per layer - size_t weights_offset = header_size + layer_info_size; - size_t weights_only_size = weights_size - weights_offset; - - WGPUBufferDescriptor buffer_desc = {}; - buffer_desc.size = weights_only_size; - buffer_desc.usage = WGPUBufferUsage_Storage | WGPUBufferUsage_CopyDst; - buffer_desc.mappedAtCreation = false; - - weights_buffer_ = wgpuDeviceCreateBuffer(ctx_.device, &buffer_desc); - - // Upload only weights (skip header + layer info) - wgpuQueueWriteBuffer(ctx_.queue, weights_buffer_, 0, - weights_data + weights_offset, weights_only_size); - - // Create uniform buffers for layer params (one per layer) - for (uint32_t i = 0; i < num_layers; ++i) { - WGPUBufferDescriptor params_desc = {}; - params_desc.size = sizeof(LayerParams); - params_desc.usage = WGPUBufferUsage_Uniform | WGPUBufferUsage_CopyDst; - params_desc.mappedAtCreation = false; - - WGPUBuffer buf = wgpuDeviceCreateBuffer(ctx_.device, ¶ms_desc); - layer_params_buffers_.push_back(buf); - } -} - -void CNNv2Effect::create_textures() { - // Static features texture (8×f16 packed as 4×u32) - TextureWithView static_tex = gpu_create_storage_texture_2d( - ctx_.device, width_, height_, WGPUTextureFormat_RGBA32Uint); - static_features_tex_ = static_tex.texture; - static_features_view_ = static_tex.view; - - // Input texture with mips (for multi-scale features) - TextureWithView input_mip = gpu_create_texture_2d( - ctx_.device, width_, height_, WGPUTextureFormat_RGBA8Unorm, - (WGPUTextureUsage)(WGPUTextureUsage_TextureBinding | - WGPUTextureUsage_CopyDst), - 3); - input_mip_tex_ = input_mip.texture; - - for (int i = 0; i < 3; ++i) { - input_mip_view_[i] = - gpu_create_mip_view(input_mip_tex_, WGPUTextureFormat_RGBA8Unorm, i); - } - - // Create 2 layer textures (ping-pong buffers for intermediate results) - // Each stores 8×f16 channels packed as 4×u32 - for (int i = 0; i < 2; ++i) { - TextureWithView layer = gpu_create_storage_texture_2d( - ctx_.device, width_, height_, WGPUTextureFormat_RGBA32Uint); - layer_textures_.push_back(layer.texture); - layer_views_.push_back(layer.view); - } - - // Create uniform buffer for static feature params - WGPUBufferDescriptor params_desc = {}; - params_desc.size = sizeof(StaticFeatureParams); - params_desc.usage = WGPUBufferUsage_Uniform | WGPUBufferUsage_CopyDst; - params_desc.mappedAtCreation = false; - static_params_buffer_ = wgpuDeviceCreateBuffer(ctx_.device, ¶ms_desc); -} - -void CNNv2Effect::create_pipelines() { - // Create linear sampler for bilinear interpolation - WGPUSamplerDescriptor sampler_desc = {}; - sampler_desc.addressModeU = WGPUAddressMode_ClampToEdge; - sampler_desc.addressModeV = WGPUAddressMode_ClampToEdge; - sampler_desc.addressModeW = WGPUAddressMode_ClampToEdge; - sampler_desc.magFilter = WGPUFilterMode_Linear; - sampler_desc.minFilter = WGPUFilterMode_Linear; - sampler_desc.mipmapFilter = WGPUMipmapFilterMode_Linear; - sampler_desc.lodMinClamp = 0.0f; - sampler_desc.lodMaxClamp = 32.0f; - sampler_desc.maxAnisotropy = 1; - - linear_sampler_ = wgpuDeviceCreateSampler(ctx_.device, &sampler_desc); - - // Static features compute pipeline - size_t shader_size = 0; - const char* static_code = - (const char*)GetAsset(AssetId::ASSET_SHADER_CNN_V2_STATIC, &shader_size); - - if (!static_code || shader_size == 0) { - // Shader not available (e.g., in test mode) - skip pipeline creation - return; - } - - WGPUShaderSourceWGSL wgsl_src = {}; - wgsl_src.chain.sType = WGPUSType_ShaderSourceWGSL; - wgsl_src.code = str_view(static_code); - - WGPUShaderModuleDescriptor shader_desc = {}; - shader_desc.nextInChain = &wgsl_src.chain; - - // Create bind group layout for static features compute - // Bindings: 0=input_tex, 1=input_mip1, 2=input_mip2, 3=depth_tex, 4=output, - // 5=params, 6=linear_sampler - WGPUBindGroupLayout static_bgl = - BindGroupLayoutBuilder() - .texture(0, WGPUShaderStage_Compute) - .texture(1, WGPUShaderStage_Compute) - .texture(2, WGPUShaderStage_Compute) - .texture(3, WGPUShaderStage_Compute) - .storage_texture(4, WGPUShaderStage_Compute, - WGPUTextureFormat_RGBA32Uint) - .uniform(5, WGPUShaderStage_Compute, sizeof(StaticFeatureParams)) - .sampler(6, WGPUShaderStage_Compute) - .build(ctx_.device); - - // Update pipeline layout - WGPUPipelineLayoutDescriptor pl_desc = {}; - pl_desc.bindGroupLayoutCount = 1; - pl_desc.bindGroupLayouts = &static_bgl; - WGPUPipelineLayout pipeline_layout = - wgpuDeviceCreatePipelineLayout(ctx_.device, &pl_desc); - - // Recreate pipeline with proper layout - WGPUComputePipelineDescriptor pipeline_desc2 = {}; - pipeline_desc2.compute.module = - wgpuDeviceCreateShaderModule(ctx_.device, &shader_desc); - pipeline_desc2.compute.entryPoint = str_view("main"); - pipeline_desc2.layout = pipeline_layout; - - if (static_pipeline_) - wgpuComputePipelineRelease(static_pipeline_); - static_pipeline_ = - wgpuDeviceCreateComputePipeline(ctx_.device, &pipeline_desc2); - - wgpuShaderModuleRelease(pipeline_desc2.compute.module); - wgpuPipelineLayoutRelease(pipeline_layout); - wgpuBindGroupLayoutRelease(static_bgl); - - // CNN layer compute pipeline (storage buffer version) - if (layer_info_.empty()) - return; // No weights loaded - - size_t layer_shader_size = 0; - const char* layer_code = (const char*)GetAsset( - AssetId::ASSET_SHADER_CNN_V2_COMPUTE, &layer_shader_size); - - if (!layer_code || layer_shader_size == 0) - return; - - WGPUShaderSourceWGSL layer_wgsl = {}; - layer_wgsl.chain.sType = WGPUSType_ShaderSourceWGSL; - layer_wgsl.code = str_view(layer_code); - - WGPUShaderModuleDescriptor layer_shader_desc = {}; - layer_shader_desc.nextInChain = &layer_wgsl.chain; - - WGPUShaderModule layer_module = - wgpuDeviceCreateShaderModule(ctx_.device, &layer_shader_desc); - if (!layer_module) - return; - - // Create bind group layout for layer compute - // 0=static_features, 1=layer_input, 2=output, 3=weights, 4=params, - // 5=original_input - WGPUBindGroupLayout layer_bgl = - BindGroupLayoutBuilder() - .uint_texture(0, WGPUShaderStage_Compute) - .uint_texture(1, WGPUShaderStage_Compute) - .storage_texture(2, WGPUShaderStage_Compute, - WGPUTextureFormat_RGBA32Uint) - .storage(3, WGPUShaderStage_Compute) - .uniform(4, WGPUShaderStage_Compute, sizeof(LayerParams)) - .texture(5, WGPUShaderStage_Compute) - .build(ctx_.device); - - WGPUPipelineLayoutDescriptor layer_pl_desc = {}; - layer_pl_desc.bindGroupLayoutCount = 1; - layer_pl_desc.bindGroupLayouts = &layer_bgl; - - WGPUPipelineLayout layer_pipeline_layout = - wgpuDeviceCreatePipelineLayout(ctx_.device, &layer_pl_desc); - - WGPUComputePipelineDescriptor layer_pipeline_desc = {}; - layer_pipeline_desc.compute.module = layer_module; - layer_pipeline_desc.compute.entryPoint = str_view("main"); - layer_pipeline_desc.layout = layer_pipeline_layout; - - layer_pipeline_ = - wgpuDeviceCreateComputePipeline(ctx_.device, &layer_pipeline_desc); - - wgpuShaderModuleRelease(layer_module); - wgpuPipelineLayoutRelease(layer_pipeline_layout); - wgpuBindGroupLayoutRelease(layer_bgl); -} - -void CNNv2Effect::update_bind_group(WGPUTextureView input_view) { - if (!static_pipeline_) - return; - - // Cache input view - current_input_view_ = input_view; - - // Release old bind group - if (static_bind_group_) { - wgpuBindGroupRelease(static_bind_group_); - static_bind_group_ = nullptr; - } - - // Create bind group for static features compute (manual for storage texture - // binding) - WGPUBindGroupEntry bg_entries[7] = {}; - bg_entries[0].binding = 0; - bg_entries[0].textureView = input_view; - bg_entries[1].binding = 1; - bg_entries[1].textureView = input_mip_view_[0]; - bg_entries[2].binding = 2; - bg_entries[2].textureView = - input_mip_view_[1] ? input_mip_view_[1] : input_mip_view_[0]; - bg_entries[3].binding = 3; - bg_entries[3].textureView = input_view; - bg_entries[4].binding = 4; - bg_entries[4].textureView = static_features_view_; - bg_entries[5].binding = 5; - bg_entries[5].buffer = static_params_buffer_; - bg_entries[5].size = sizeof(StaticFeatureParams); - bg_entries[6].binding = 6; - bg_entries[6].sampler = linear_sampler_; - - WGPUBindGroupLayout layout = - wgpuComputePipelineGetBindGroupLayout(static_pipeline_, 0); - WGPUBindGroupDescriptor bg_desc = {}; - bg_desc.layout = layout; - bg_desc.entryCount = 7; - bg_desc.entries = bg_entries; - static_bind_group_ = wgpuDeviceCreateBindGroup(ctx_.device, &bg_desc); - wgpuBindGroupLayoutRelease(layout); - - // Create layer bind groups - if (!layer_pipeline_ || layer_info_.empty()) - return; - - // Release old layer bind groups - for (auto bg : layer_bind_groups_) { - wgpuBindGroupRelease(bg); - } - layer_bind_groups_.clear(); - - // Get bind group layout from layer pipeline - WGPUBindGroupLayout layer_bgl = - wgpuComputePipelineGetBindGroupLayout(layer_pipeline_, 0); - - // Create bind group for each layer - for (size_t i = 0; i < layer_info_.size(); ++i) { - WGPUTextureView layer_input = - (i == 0) ? static_features_view_ : layer_views_[i % 2]; - - WGPUBindGroup layer_bg = - BindGroupBuilder() - .texture(0, static_features_view_) - .texture(1, layer_input) - .texture(2, layer_views_[(i + 1) % 2]) - .buffer(3, weights_buffer_, wgpuBufferGetSize(weights_buffer_)) - .buffer(4, layer_params_buffers_[i], sizeof(LayerParams)) - .texture(5, input_view) - .build(ctx_.device, layer_bgl); - - layer_bind_groups_.push_back(layer_bg); - } - - wgpuBindGroupLayoutRelease(layer_bgl); -} - -void CNNv2Effect::compute(WGPUCommandEncoder encoder, - const CommonPostProcessUniforms& uniforms) { - if (!initialized_ || !static_pipeline_ || !static_bind_group_) - return; - - float effective_blend = blend_amount_; - if (beat_modulated_) { - effective_blend = blend_amount_ * uniforms.beat_phase * beat_scale_; - } - - // Update static feature params - StaticFeatureParams static_params; - static_params.mip_level = mip_level_; - static_params.padding[0] = 0; - static_params.padding[1] = 0; - static_params.padding[2] = 0; - wgpuQueueWriteBuffer(ctx_.queue, static_params_buffer_, 0, &static_params, - sizeof(static_params)); - - // Pass 1: Compute static features - WGPUComputePassEncoder pass = - wgpuCommandEncoderBeginComputePass(encoder, nullptr); - - wgpuComputePassEncoderSetPipeline(pass, static_pipeline_); - wgpuComputePassEncoderSetBindGroup(pass, 0, static_bind_group_, 0, nullptr); - - // Dispatch workgroups (8×8 threads per group) - uint32_t workgroups_x = (width_ + 7) / 8; - uint32_t workgroups_y = (height_ + 7) / 8; - wgpuComputePassEncoderDispatchWorkgroups(pass, workgroups_x, workgroups_y, 1); - - wgpuComputePassEncoderEnd(pass); - wgpuComputePassEncoderRelease(pass); - - // Execute CNN layer passes - if (!layer_pipeline_ || layer_bind_groups_.empty()) - return; - - // Update layer params (each layer has own buffer) - for (size_t i = 0; i < layer_info_.size(); ++i) { - const LayerInfo& info = layer_info_[i]; - - LayerParams params; - params.kernel_size = info.kernel_size; - params.in_channels = info.in_channels; - params.out_channels = info.out_channels; - params.weight_offset = info.weight_offset; - params.is_output_layer = (i == layer_info_.size() - 1) ? 1 : 0; - params.blend_amount = effective_blend; - params.is_layer_0 = (i == 0) ? 1 : 0; - - wgpuQueueWriteBuffer(ctx_.queue, layer_params_buffers_[i], 0, ¶ms, - sizeof(params)); - - WGPUComputePassEncoder layer_pass = - wgpuCommandEncoderBeginComputePass(encoder, nullptr); - - wgpuComputePassEncoderSetPipeline(layer_pass, layer_pipeline_); - wgpuComputePassEncoderSetBindGroup(layer_pass, 0, layer_bind_groups_[i], 0, - nullptr); - - wgpuComputePassEncoderDispatchWorkgroups(layer_pass, workgroups_x, - workgroups_y, 1); - - wgpuComputePassEncoderEnd(layer_pass); - wgpuComputePassEncoderRelease(layer_pass); - } -} - -void CNNv2Effect::render(WGPURenderPassEncoder pass, - const CommonPostProcessUniforms& uniforms) { - (void)pass; - (void)uniforms; - // Compute-only effect, rendering is done by default composite pass -} - -void CNNv2Effect::cleanup() { - if (static_features_view_) - wgpuTextureViewRelease(static_features_view_); - if (static_features_tex_) - wgpuTextureRelease(static_features_tex_); - if (static_bind_group_) - wgpuBindGroupRelease(static_bind_group_); - if (static_params_buffer_) - wgpuBufferRelease(static_params_buffer_); - if (static_pipeline_) - wgpuComputePipelineRelease(static_pipeline_); - if (linear_sampler_) - wgpuSamplerRelease(linear_sampler_); - - if (layer_pipeline_) - wgpuComputePipelineRelease(layer_pipeline_); - if (weights_buffer_) - wgpuBufferRelease(weights_buffer_); - for (auto buf : layer_params_buffers_) - wgpuBufferRelease(buf); - layer_params_buffers_.clear(); - - for (int i = 0; i < 3; ++i) { - if (input_mip_view_[i]) - wgpuTextureViewRelease(input_mip_view_[i]); - } - if (input_mip_tex_) - wgpuTextureRelease(input_mip_tex_); - - for (auto view : layer_views_) - wgpuTextureViewRelease(view); - for (auto tex : layer_textures_) - wgpuTextureRelease(tex); - for (auto bg : layer_bind_groups_) - wgpuBindGroupRelease(bg); - - layer_views_.clear(); - layer_textures_.clear(); - layer_bind_groups_.clear(); - layer_info_.clear(); - - initialized_ = false; -} diff --git a/src/effects/cnn_v2_effect.h b/src/effects/cnn_v2_effect.h deleted file mode 100644 index 7960b4f..0000000 --- a/src/effects/cnn_v2_effect.h +++ /dev/null @@ -1,89 +0,0 @@ -// CNN v2 Effect - Parametric Static Features -// Multi-pass post-processing with 7D feature input -// Supports per-layer kernel sizes (e.g., 1×1, 3×3, 5×5) - -#pragma once -#include "gpu/effect.h" -#include <vector> - -struct CNNv2EffectParams { - float blend_amount = 1.0f; -}; - -class CNNv2Effect : public PostProcessEffect { - public: - explicit CNNv2Effect(const GpuContext& ctx); - explicit CNNv2Effect(const GpuContext& ctx, const CNNv2EffectParams& params); - ~CNNv2Effect(); - - void init(MainSequence* demo) override; - void resize(int width, int height) override; - void compute(WGPUCommandEncoder encoder, - const CommonPostProcessUniforms& uniforms) override; - void render(WGPURenderPassEncoder pass, - const CommonPostProcessUniforms& uniforms) override; - void update_bind_group(WGPUTextureView input_view) override; - - void set_beat_modulation(bool enabled, float scale = 1.0f) { - beat_modulated_ = enabled; - beat_scale_ = scale; - } - - private: - struct LayerInfo { - uint32_t kernel_size; - uint32_t in_channels; - uint32_t out_channels; - uint32_t weight_offset; - uint32_t weight_count; - }; - - struct LayerParams { - uint32_t kernel_size; - uint32_t in_channels; - uint32_t out_channels; - uint32_t weight_offset; - uint32_t is_output_layer; - float blend_amount; - uint32_t is_layer_0; - }; - - struct StaticFeatureParams { - uint32_t mip_level; - uint32_t padding[3]; - }; - - void create_textures(); - void create_pipelines(); - void load_weights(); - void cleanup(); - - // Static features compute - WGPUComputePipeline static_pipeline_; - WGPUBindGroup static_bind_group_; - WGPUBuffer static_params_buffer_; - WGPUTexture static_features_tex_; - WGPUTextureView static_features_view_; - WGPUSampler linear_sampler_; - - // CNN layers (storage buffer architecture) - WGPUComputePipeline layer_pipeline_; // Single pipeline for all layers - WGPUBuffer weights_buffer_; // Storage buffer for weights - std::vector<WGPUBuffer> - layer_params_buffers_; // Uniform buffers (one per layer) - std::vector<LayerInfo> layer_info_; // Layer metadata - std::vector<WGPUBindGroup> layer_bind_groups_; // Per-layer bind groups - std::vector<WGPUTexture> layer_textures_; // Ping-pong buffers - std::vector<WGPUTextureView> layer_views_; - - // Input mips - WGPUTexture input_mip_tex_; - WGPUTextureView input_mip_view_[3]; - WGPUTextureView current_input_view_; - - float blend_amount_ = 1.0f; - bool beat_modulated_ = false; - float beat_scale_ = 1.0f; - uint32_t mip_level_ = 0; - bool initialized_; -}; diff --git a/src/effects/flash_cube_effect.cc b/src/effects/flash_cube_effect.cc index 29e9897..383e66a 100644 --- a/src/effects/flash_cube_effect.cc +++ b/src/effects/flash_cube_effect.cc @@ -60,12 +60,12 @@ void FlashCubeEffect::render(WGPURenderPassEncoder pass, // Detect beat changes for flash trigger (using intensity as proxy for beat // hits) Intensity spikes on beats, so we can use it to trigger flashes if (uniforms.audio_intensity > 0.5f && - flash_intensity_ < 0.3f) { // High intensity + flash cooled down + flash_intensity_ < 0.2f) { // High intensity + flash cooled down flash_intensity_ = 1.0f; // Trigger full flash } // Exponential decay of flash - flash_intensity_ *= 0.90f; // Slower fade for more visible effect + flash_intensity_ *= 0.95f; // Slower fade for more visible effect // Always have base brightness, add flash on top float base_brightness = 0.2f; @@ -80,7 +80,7 @@ void FlashCubeEffect::render(WGPURenderPassEncoder pass, // Slowly rotate the cube for visual interest scene_.objects[0].rotation = - quat::from_axis(vec3(0.3f, 1, 0.2f), uniforms.time * 0.05f); + quat::from_axis(vec3(0.3f, 1, 0.2f), uniforms.time * 0.04f); // Position camera OUTSIDE the cube looking at it from a distance // This way we see the cube as a background element diff --git a/src/effects/gaussian_blur_effect.h b/src/effects/gaussian_blur_effect.h index 651c5c3..bf1062f 100644 --- a/src/effects/gaussian_blur_effect.h +++ b/src/effects/gaussian_blur_effect.h @@ -8,9 +8,9 @@ // Parameters for GaussianBlurEffect (set at construction time) struct GaussianBlurParams { - float strength = 1.0f; // Default + float strength = 1.0f; // Default float strength_audio = 0.5f; // how much to pulse with audio - float stretch = 1.f; // y/x axis ratio + float stretch = 1.f; // y/x axis ratio float _pad = 0.; }; static_assert(sizeof(GaussianBlurParams) == 16, diff --git a/src/effects/particle_spray_effect.h b/src/effects/particle_spray_effect.h index c83d691..216e13f 100644 --- a/src/effects/particle_spray_effect.h +++ b/src/effects/particle_spray_effect.h @@ -3,8 +3,8 @@ #pragma once -#include "gpu/effect.h" #include "effects/particle_defs.h" +#include "gpu/effect.h" class ParticleSprayEffect : public Effect { public: diff --git a/src/effects/particles_effect.h b/src/effects/particles_effect.h index 6d46ea2..a69039f 100644 --- a/src/effects/particles_effect.h +++ b/src/effects/particles_effect.h @@ -3,8 +3,8 @@ #pragma once -#include "gpu/effect.h" #include "effects/particle_defs.h" +#include "gpu/effect.h" class ParticlesEffect : public Effect { public: diff --git a/src/effects/sdf_test_effect.cc b/src/effects/sdf_test_effect.cc index 28b3513..264809f 100644 --- a/src/effects/sdf_test_effect.cc +++ b/src/effects/sdf_test_effect.cc @@ -9,8 +9,8 @@ SDFTestEffect::SDFTestEffect(const GpuContext& ctx) : SDFEffect(ctx) { ResourceBinding bindings[] = { {uniforms_.get(), WGPUBufferBindingType_Uniform}, {camera_params_.get(), WGPUBufferBindingType_Uniform}}; - pass_ = gpu_create_render_pass(ctx_.device, ctx_.format, - sdf_test_shader_wgsl, bindings, 2); + pass_ = gpu_create_render_pass(ctx_.device, ctx_.format, sdf_test_shader_wgsl, + bindings, 2); pass_.vertex_count = 3; } diff --git a/src/gpu/demo_effects.h b/src/gpu/demo_effects.h index 85498ad..6b22f3f 100644 --- a/src/gpu/demo_effects.h +++ b/src/gpu/demo_effects.h @@ -18,8 +18,8 @@ // Individual Effect Headers #include "effects/chroma_aberration_effect.h" #include "effects/circle_mask_effect.h" -#include "effects/cnn_effect.h" -#include "effects/cnn_v2_effect.h" +#include "../../cnn_v1/src/cnn_v1_effect.h" +#include "../../cnn_v2/src/cnn_v2_effect.h" #include "effects/distort_effect.h" #include "effects/fade_effect.h" #include "effects/flash_cube_effect.h" @@ -40,12 +40,8 @@ #include <memory> - - // Common particle definition is now in effects/particle_defs.h - - // Auto-generated functions from sequence compiler void LoadTimeline(MainSequence& main_seq, const GpuContext& ctx); diff --git a/src/gpu/gpu.cc b/src/gpu/gpu.cc index ce234fa..ff4def7 100644 --- a/src/gpu/gpu.cc +++ b/src/gpu/gpu.cc @@ -143,7 +143,6 @@ RenderPass gpu_create_render_pass(WGPUDevice device, WGPUTextureFormat format, WGPUShaderModule shader_module = wgpuDeviceCreateShaderModule(device, &shader_desc); - // Create Bind Group Layout & Bind Group std::vector<WGPUBindGroupLayoutEntry> bgl_entries; std::vector<WGPUBindGroupEntry> bg_entries; diff --git a/src/tests/audio/test_audio_engine.cc b/src/tests/audio/test_audio_engine.cc index 3f0ad4d..3d23a5c 100644 --- a/src/tests/audio/test_audio_engine.cc +++ b/src/tests/audio/test_audio_engine.cc @@ -65,20 +65,17 @@ void test_audio_engine_manual_resource_loading() { // Manually preload first few samples res_mgr->preload(0); res_mgr->preload(1); - res_mgr->preload(2); const int after_preload = res_mgr->get_loaded_count(); printf(" Samples loaded after manual preload: %d\n", after_preload); - assert(after_preload == 3); // Should have 3 samples loaded + assert(after_preload == 2); // Should have 2 samples loaded // Verify samples are accessible const Spectrogram* spec0 = res_mgr->get_spectrogram(0); const Spectrogram* spec1 = res_mgr->get_spectrogram(1); - const Spectrogram* spec2 = res_mgr->get_spectrogram(2); assert(spec0 != nullptr); assert(spec1 != nullptr); - assert(spec2 != nullptr); printf(" ✓ AudioEngine manual resource loading test passed\n"); } @@ -97,10 +94,9 @@ void test_audio_engine_reset() { // Manually load some samples res_mgr->preload(0); res_mgr->preload(1); - res_mgr->preload(2); const int loaded_before_reset = res_mgr->get_loaded_count(); - assert(loaded_before_reset == 3); + assert(loaded_before_reset == 2); // Reset engine fixture.engine().reset(); diff --git a/src/tests/gpu/test_demo_effects.cc b/src/tests/gpu/test_demo_effects.cc index ec78c10..8726e55 100644 --- a/src/tests/gpu/test_demo_effects.cc +++ b/src/tests/gpu/test_demo_effects.cc @@ -12,7 +12,7 @@ #include "../common/effect_test_helpers.h" #include "../common/webgpu_test_fixture.h" -#include "effects/cnn_effect.h" +#include "../../../cnn_v1/src/cnn_v1_effect.h" #include "gpu/demo_effects.h" #include "gpu/effect.h" #include <cassert> @@ -89,7 +89,7 @@ static void test_post_process_effects() { {"ThemeModulationEffect", std::make_shared<ThemeModulationEffect>(fixture.ctx())}, {"VignetteEffect", std::make_shared<VignetteEffect>(fixture.ctx())}, - {"CNNEffect", std::make_shared<CNNEffect>(fixture.ctx())}, + {"CNNv1Effect", std::make_shared<CNNv1Effect>(fixture.ctx())}, {"CNNv2Effect", std::make_shared<CNNv2Effect>(fixture.ctx())}, }; diff --git a/src/tests/gpu/test_shader_assets.cc b/src/tests/gpu/test_shader_assets.cc index 135c477..63f9b5d 100644 --- a/src/tests/gpu/test_shader_assets.cc +++ b/src/tests/gpu/test_shader_assets.cc @@ -42,8 +42,8 @@ int main() { all_passed &= validate_shader(AssetId::ASSET_SHADER_COMMON_UNIFORMS, "COMMON_UNIFORMS", {"struct", "GlobalUniforms"}); - all_passed &= validate_shader(AssetId::ASSET_SHADER_SDF_SHAPES, - "SDF_SHAPES", {"fn", "sd"}); + all_passed &= validate_shader(AssetId::ASSET_SHADER_SDF_SHAPES, "SDF_SHAPES", + {"fn", "sd"}); all_passed &= validate_shader(AssetId::ASSET_SHADER_LIGHTING, "LIGHTING", {"fn", "calc"}); all_passed &= validate_shader(AssetId::ASSET_SHADER_RAY_BOX, "RAY_BOX", |
