From 2e34965e5e48175c5ee6016af1a858f7f3f02c1d Mon Sep 17 00:00:00 2001 From: skal Date: Wed, 4 Feb 2026 11:56:03 +0100 Subject: feat(gpu): Implement recursive WGSL composition and modularize shaders (Task #50) - Updated ShaderComposer to support recursive #include "snippet_name" with cycle detection. - Extracted granular WGSL snippets: math/sdf_shapes, math/sdf_utils, render/shadows, render/scene_query, render/lighting_utils. - Refactored Renderer3D to use #include in shaders, simplifying C++ dependency lists. - Fixed WGPUShaderSourceWGSL usage on macOS to correctly handle composed shader strings. - Added comprehensive unit tests for recursive composition in test_shader_composer. - Verified system stability with test_3d_render and full test suite. - Marked Task #50 as recurrent for future code hygiene. --- TODO.md | 27 ++++-------- assets/final/demo_assets.txt | 7 +++- assets/final/shaders/math/sdf_shapes.wgsl | 14 +++++++ assets/final/shaders/math/sdf_utils.wgsl | 9 ++++ assets/final/shaders/render/lighting_utils.wgsl | 6 +++ assets/final/shaders/render/scene_query.wgsl | 33 +++++++++++++++ assets/final/shaders/render/shadows.wgsl | 13 ++++++ assets/final/shaders/renderer_3d.wgsl | 44 ++++---------------- assets/final/shaders/skybox.wgsl | 2 + src/3d/renderer.cc | 15 ++++--- src/gpu/effects/shader_composer.cc | 48 ++++++++++++++++++--- src/gpu/effects/shader_composer.h | 6 +++ src/gpu/effects/shaders.cc | 5 +++ src/tests/test_shader_composer.cc | 55 +++++++++++++++++++++++-- 14 files changed, 212 insertions(+), 72 deletions(-) create mode 100644 assets/final/shaders/math/sdf_shapes.wgsl create mode 100644 assets/final/shaders/math/sdf_utils.wgsl create mode 100644 assets/final/shaders/render/lighting_utils.wgsl create mode 100644 assets/final/shaders/render/scene_query.wgsl create mode 100644 assets/final/shaders/render/shadows.wgsl diff --git a/TODO.md b/TODO.md index 6a02db7..9477c68 100644 --- a/TODO.md +++ b/TODO.md @@ -3,6 +3,12 @@ This file tracks prioritized tasks with detailed attack plans. ## Recently Completed (February 4, 2026) +- [x] **Task #50: WGSL Modularization**: + - [x] **Recursive Composition**: Updated `ShaderComposer` to support recursive `#include "snippet_name"` directives with cycle detection. + - [x] **Granular SDF Library**: Extracted `math/sdf_shapes.wgsl`, `math/sdf_utils.wgsl`, `render/shadows.wgsl`, `render/scene_query.wgsl`, and `render/lighting_utils.wgsl`. + - [x] **Pipeline Update**: Refactored `Renderer3D` and `renderer_3d.wgsl` to use the new modular system, reducing C++-side dependency management. + - [x] **Platform Fix**: Resolved `WGPUShaderSourceWGSL` usage on macOS to ensure compatibility with composed shader strings. + - [x] **Task #48: Improve Audio Coverage**: - [x] **New Tests**: Added `test_dct` (100% coverage for transforms) and `test_audio_gen` (94% coverage for procedural audio). - [x] **Enhanced Tests**: Updated `test_synth` to cover rendering loop, double-buffering, and resource exhaustion. @@ -30,25 +36,8 @@ This file tracks prioritized tasks with detailed attack plans. - [ ] **Task #37: Asset Ingestion:** Update `asset_packer` to handle the new 3D binary format. - [ ] **Task #38: Runtime Loader:** Implement a minimal C++ parser to load the scene data into the ECS/Renderer. -## Priority 2: WGSL Modularization (Task #50) -**Goal**: Refactor `ShaderComposer` and WGSL assets to support granular, reusable snippets and `#include` directives. - -- [ ] **Task #50.1: Recursive Composition Support**: - - [ ] Update `ShaderComposer::Compose` to parse `#include "snippet_name"` directives in shader code. - - [ ] Implement recursive resolution (with cycle detection) to assemble the final shader. - - [ ] Update `test_shader_composer.cc` to verify recursive includes. - -- [ ] **Task #50.2: Granular SDF Library**: - - [ ] Extract `sdSphere`, `sdBox`, `sdTorus`, `sdPlane` from `sdf_primitives.wgsl` into `math/sdf_shapes.wgsl`. - - [ ] Extract `get_normal_basic` into `math/sdf_utils.wgsl`. - - [ ] Extract `calc_shadow` into `render/shadows.wgsl`. - -- [ ] **Task #50.3: Scene & Material Refactor**: - - [ ] Extract `map_scene` and `get_dist` logic into `render/scene_query.wgsl`. - - [ ] Extract lighting and material logic from `fs_main` into `render/lighting_pbr.wgsl` (or similar). - -- [ ] **Task #50.4: Pipeline Update**: - - [ ] Update `Renderer3D` to use the new granular assets via `#include` in the main shader, reducing C++-side dependency lists. +## Priority 2: WGSL Modularization (Task #50) [RECURRENT] +**Goal**: Refactor `ShaderComposer` and WGSL assets to support granular, reusable snippets and `#include` directives. This is an ongoing task to maintain shader code hygiene as new features are added. ## Priority 3: Physics & Collision (Task #49) **Goal**: Implement a lightweight physics engine using SDFs and BVH acceleration. (See `doc/3D.md` for design). diff --git a/assets/final/demo_assets.txt b/assets/final/demo_assets.txt index a1060cb..5ba2ec2 100644 --- a/assets/final/demo_assets.txt +++ b/assets/final/demo_assets.txt @@ -40,4 +40,9 @@ SHADER_SOLARIZE, NONE, shaders/solarize.wgsl, "Solarize Shader" SHADER_DISTORT, NONE, shaders/distort.wgsl, "Distort Shader" SHADER_CHROMA_ABERRATION, NONE, shaders/chroma_aberration.wgsl, "Chroma Aberration Shader" SHADER_VISUAL_DEBUG, NONE, shaders/visual_debug.wgsl, "Visual Debug Shader" -SHADER_SKYBOX, NONE, shaders/skybox.wgsl, "Skybox background shader" \ No newline at end of file +SHADER_SKYBOX, NONE, shaders/skybox.wgsl, "Skybox background shader" +SHADER_MATH_SDF_SHAPES, NONE, shaders/math/sdf_shapes.wgsl, "SDF Shapes Snippet" +SHADER_MATH_SDF_UTILS, NONE, shaders/math/sdf_utils.wgsl, "SDF Utils Snippet" +SHADER_RENDER_SHADOWS, NONE, shaders/render/shadows.wgsl, "Shadows Snippet" +SHADER_RENDER_SCENE_QUERY, NONE, shaders/render/scene_query.wgsl, "Scene Query Snippet" +SHADER_RENDER_LIGHTING_UTILS, NONE, shaders/render/lighting_utils.wgsl, "Lighting Utils Snippet" \ No newline at end of file diff --git a/assets/final/shaders/math/sdf_shapes.wgsl b/assets/final/shaders/math/sdf_shapes.wgsl new file mode 100644 index 0000000..31bbe2d --- /dev/null +++ b/assets/final/shaders/math/sdf_shapes.wgsl @@ -0,0 +1,14 @@ +fn sdSphere(p: vec3, r: f32) -> f32 { + return length(p) - r; +} +fn sdBox(p: vec3, b: vec3) -> f32 { + let q = abs(p) - b; + return length(max(q, vec3(0.0))) + min(max(q.x, max(q.y, q.z)), 0.0); +} +fn sdTorus(p: vec3, t: vec2) -> f32 { + let q = vec2(length(p.xz) - t.x, p.y); + return length(q) - t.y; +} +fn sdPlane(p: vec3, n: vec3, h: f32) -> f32 { + return dot(p, n) + h; +} diff --git a/assets/final/shaders/math/sdf_utils.wgsl b/assets/final/shaders/math/sdf_utils.wgsl new file mode 100644 index 0000000..ce902bf --- /dev/null +++ b/assets/final/shaders/math/sdf_utils.wgsl @@ -0,0 +1,9 @@ +fn get_normal_basic(p: vec3, obj_type: f32) -> vec3 { + if (obj_type == 1.0) { return normalize(p); } + let e = vec2(0.001, 0.0); + return normalize(vec3( + get_dist(p + e.xyy, obj_type) - get_dist(p - e.xyy, obj_type), + get_dist(p + e.yxy, obj_type) - get_dist(p - e.yxy, obj_type), + get_dist(p + e.yyx, obj_type) - get_dist(p - e.yyx, obj_type) + )); +} diff --git a/assets/final/shaders/render/lighting_utils.wgsl b/assets/final/shaders/render/lighting_utils.wgsl new file mode 100644 index 0000000..d2fd2e2 --- /dev/null +++ b/assets/final/shaders/render/lighting_utils.wgsl @@ -0,0 +1,6 @@ +fn calculate_lighting(color: vec3, normal: vec3, pos: vec3, shadow: f32) -> vec3 { + let light_dir = normalize(vec3(1.0, 1.0, 1.0)); + let diffuse = max(dot(normal, light_dir), 0.0); + let lighting = diffuse * (0.1 + 0.9 * shadow) + 0.1; // Ambient + Shadowed Diffuse + return color * lighting; +} diff --git a/assets/final/shaders/render/scene_query.wgsl b/assets/final/shaders/render/scene_query.wgsl new file mode 100644 index 0000000..b58f28c --- /dev/null +++ b/assets/final/shaders/render/scene_query.wgsl @@ -0,0 +1,33 @@ +#include "math/sdf_shapes" + +fn get_dist(p: vec3, obj_type: f32) -> f32 { + if (obj_type == 1.0) { return length(p) - 1.0; } // Unit Sphere + if (obj_type == 2.0) { return sdBox(p, vec3(1.0)); } // Unit Box + if (obj_type == 3.0) { return sdTorus(p, vec2(1.0, 0.4)); } // Unit Torus + if (obj_type == 4.0) { return sdPlane(p, vec3(0.0, 1.0, 0.0), 0.0); } + return 100.0; +} + +fn map_scene(p: vec3, skip_idx: u32) -> f32 { + var d = 1000.0; + let count = u32(globals.params.x); + + for (var i = 0u; i < count; i = i + 1u) { + if (i == skip_idx) { continue; } + let obj = object_data.objects[i]; + let obj_type = obj.params.x; + // Skip rasterized objects (like the floor) in the SDF map + if (obj_type <= 0.0) { continue; } + + let q = (obj.inv_model * vec4(p, 1.0)).xyz; + + let scale_x = length(obj.model[0].xyz); + let scale_y = length(obj.model[1].xyz); + let scale_z = length(obj.model[2].xyz); + // Use conservative minimum scale to avoid overstepping the distance field + let s = min(scale_x, min(scale_y, scale_z)); + + d = min(d, get_dist(q, obj_type) * s); + } + return d; +} diff --git a/assets/final/shaders/render/shadows.wgsl b/assets/final/shaders/render/shadows.wgsl new file mode 100644 index 0000000..7cba089 --- /dev/null +++ b/assets/final/shaders/render/shadows.wgsl @@ -0,0 +1,13 @@ +fn calc_shadow(ro: vec3, rd: vec3, tmin: f32, tmax: f32, skip_idx: u32) -> f32 { + var res = 1.0; + var t = tmin; + if (t < 0.05) { t = 0.05; } + for (var i = 0; i < 32; i = i + 1) { + let h = map_scene(ro + rd * t, skip_idx); + if (h < 0.001) { return 0.0; } + res = min(res, 16.0 * h / t); + t = t + clamp(h, 0.02, 0.4); + if (t > tmax) { break; } + } + return clamp(res, 0.0, 1.0); +} diff --git a/assets/final/shaders/renderer_3d.wgsl b/assets/final/shaders/renderer_3d.wgsl index 7be8d4e..b39525d 100644 --- a/assets/final/shaders/renderer_3d.wgsl +++ b/assets/final/shaders/renderer_3d.wgsl @@ -1,3 +1,5 @@ +#include "common_uniforms" + @group(0) @binding(0) var globals: GlobalUniforms; @group(0) @binding(1) var object_data: ObjectsBuffer; @group(0) @binding(2) var noise_tex: texture_2d; @@ -54,37 +56,10 @@ fn vs_main(@builtin(vertex_index) vertex_index: u32, return out; } -fn get_dist(p: vec3, obj_type: f32) -> f32 { - if (obj_type == 1.0) { return length(p) - 1.0; } // Unit Sphere - if (obj_type == 2.0) { return sdBox(p, vec3(1.0)); } // Unit Box - if (obj_type == 3.0) { return sdTorus(p, vec2(1.0, 0.4)); } // Unit Torus - if (obj_type == 4.0) { return sdPlane(p, vec3(0.0, 1.0, 0.0), 0.0); } - return 100.0; -} - -fn map_scene(p: vec3, skip_idx: u32) -> f32 { - var d = 1000.0; - let count = u32(globals.params.x); - - for (var i = 0u; i < count; i = i + 1u) { - if (i == skip_idx) { continue; } - let obj = object_data.objects[i]; - let obj_type = obj.params.x; - // Skip rasterized objects (like the floor) in the SDF map - if (obj_type <= 0.0) { continue; } - - let q = (obj.inv_model * vec4(p, 1.0)).xyz; - - let scale_x = length(obj.model[0].xyz); - let scale_y = length(obj.model[1].xyz); - let scale_z = length(obj.model[2].xyz); - // Use conservative minimum scale to avoid overstepping the distance field - let s = min(scale_x, min(scale_y, scale_z)); - - d = min(d, get_dist(q, obj_type) * s); - } - return d; -} +#include "render/scene_query" +#include "render/shadows" +#include "render/lighting_utils" +#include "ray_box" @fragment fn fs_main(in: VertexOutput) -> @location(0) vec4 { @@ -189,7 +164,6 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { } let shadow = calc_shadow(p, light_dir, 0.05, 20.0, in.instance_index); - let diffuse = max(dot(normal, light_dir), 0.0); - let lighting = diffuse * (0.1 + 0.9 * shadow) + 0.1; // Ambient + Shadowed Diffuse - return vec4(base_color * lighting, 1.0); -} \ No newline at end of file + let lit_color = calculate_lighting(base_color, normal, p, shadow); + return vec4(lit_color, 1.0); +} diff --git a/assets/final/shaders/skybox.wgsl b/assets/final/shaders/skybox.wgsl index 6becc1a..d7f252e 100644 --- a/assets/final/shaders/skybox.wgsl +++ b/assets/final/shaders/skybox.wgsl @@ -1,3 +1,5 @@ +#include "common_uniforms" + @group(0) @binding(0) var sky_tex: texture_2d; @group(0) @binding(1) var sky_sampler: sampler; @group(0) @binding(2) var globals: GlobalUniforms; diff --git a/src/3d/renderer.cc b/src/3d/renderer.cc index 778509f..e4320f4 100644 --- a/src/3d/renderer.cc +++ b/src/3d/renderer.cc @@ -66,8 +66,8 @@ void Renderer3D::create_skybox_pipeline() { const uint8_t* shader_code_asset = GetAsset(AssetId::ASSET_SHADER_SKYBOX, nullptr); - std::string shader_source = ShaderComposer::Get().Compose( - {"common_uniforms"}, (const char*)shader_code_asset); + std::string shader_source = + ShaderComposer::Get().Compose({}, (const char*)shader_code_asset); #if defined(DEMO_CROSS_COMPILE_WIN32) WGPUShaderModuleWGSLDescriptor wgsl_desc = {}; @@ -78,7 +78,7 @@ void Renderer3D::create_skybox_pipeline() { #else WGPUShaderSourceWGSL wgsl_desc = {}; wgsl_desc.chain.sType = WGPUSType_ShaderSourceWGSL; - wgsl_desc.code = {shader_source.c_str(), shader_source.length()}; + wgsl_desc.code = str_view(shader_source.c_str()); WGPUShaderModuleDescriptor shader_desc = {}; shader_desc.nextInChain = (const WGPUChainedStruct*)&wgsl_desc.chain; #endif @@ -236,10 +236,9 @@ void Renderer3D::create_pipeline() { WGPUPipelineLayout pipeline_layout = wgpuDeviceCreatePipelineLayout(device_, &pl_desc); - std::string main_code = - (const char*)GetAsset(AssetId::ASSET_SHADER_RENDERER_3D); - std::string shader_source = ShaderComposer::Get().Compose( - {"common_uniforms", "sdf_primitives", "lighting", "ray_box"}, main_code); + const char* asset_data = (const char*)GetAsset(AssetId::ASSET_SHADER_RENDERER_3D); + std::string main_code = asset_data; + std::string shader_source = ShaderComposer::Get().Compose({}, main_code); #if defined(DEMO_CROSS_COMPILE_WIN32) WGPUShaderModuleWGSLDescriptor wgsl_desc = {}; @@ -250,7 +249,7 @@ void Renderer3D::create_pipeline() { #else WGPUShaderSourceWGSL wgsl_desc = {}; wgsl_desc.chain.sType = WGPUSType_ShaderSourceWGSL; - wgsl_desc.code = {shader_source.c_str(), shader_source.length()}; + wgsl_desc.code = str_view(shader_source.c_str()); WGPUShaderModuleDescriptor shader_desc = {}; shader_desc.nextInChain = (const WGPUChainedStruct*)&wgsl_desc.chain; #endif diff --git a/src/gpu/effects/shader_composer.cc b/src/gpu/effects/shader_composer.cc index 3e08df9..8d66ad7 100644 --- a/src/gpu/effects/shader_composer.cc +++ b/src/gpu/effects/shader_composer.cc @@ -2,6 +2,7 @@ // It implements the ShaderComposer class. #include "gpu/effects/shader_composer.h" +#include #include ShaderComposer& ShaderComposer::Get() { @@ -14,22 +15,59 @@ void ShaderComposer::RegisterSnippet(const std::string& name, snippets_[name] = code; } +void ShaderComposer::ResolveRecursive(const std::string& source, + std::stringstream& ss, + std::set& included) { + std::istringstream stream(source); + std::string line; + while (std::getline(stream, line)) { + // Check for #include "snippet_name" + if (line.compare(0, 9, "#include ") == 0) { + size_t start = line.find('"'); + size_t end = line.find('"', start + 1); + if (start != std::string::npos && end != std::string::npos) { + std::string name = line.substr(start + 1, end - start - 1); + if (included.find(name) == included.end()) { + included.insert(name); + auto it = snippets_.find(name); + if (it != snippets_.end()) { + ss << "// --- Included: " << name << " ---\n"; + ResolveRecursive(it->second, ss, included); + ss << "// --- End Include: " << name << " ---\n"; + } else { + ss << "// ERROR: Snippet not found: " << name << "\n"; + } + } + } + } else { + ss << line << "\n"; + } + } +} + std::string ShaderComposer::Compose(const std::vector& dependencies, const std::string& main_code) { std::stringstream ss; ss << "// Generated by ShaderComposer\n\n"; + std::set included; + + // Process explicit dependencies first for (const auto& dep : dependencies) { - auto it = snippets_.find(dep); - if (it != snippets_.end()) { - ss << "// --- Snippet: " << dep << " ---\n"; - ss << it->second << "\n"; + if (included.find(dep) == included.end()) { + included.insert(dep); + auto it = snippets_.find(dep); + if (it != snippets_.end()) { + ss << "// --- Dependency: " << dep << " ---\n"; + ResolveRecursive(it->second, ss, included); + ss << "\n"; + } } } ss << "// --- Main Code ---\n"; - ss << main_code; + ResolveRecursive(main_code, ss, included); return ss.str(); } diff --git a/src/gpu/effects/shader_composer.h b/src/gpu/effects/shader_composer.h index 49bf00c..a63a6a4 100644 --- a/src/gpu/effects/shader_composer.h +++ b/src/gpu/effects/shader_composer.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include @@ -15,10 +16,15 @@ class ShaderComposer { void RegisterSnippet(const std::string& name, const std::string& code); // Assemble a final shader string by prepending required snippets + // and recursively resolving #include "snippet_name" directives. std::string Compose(const std::vector& dependencies, const std::string& main_code); private: ShaderComposer() = default; + + void ResolveRecursive(const std::string& source, std::stringstream& ss, + std::set& included); + std::map snippets_; }; diff --git a/src/gpu/effects/shaders.cc b/src/gpu/effects/shaders.cc index b2d184d..7255cf5 100644 --- a/src/gpu/effects/shaders.cc +++ b/src/gpu/effects/shaders.cc @@ -31,6 +31,11 @@ void InitShaderComposer() { }; register_if_exists("common_uniforms", AssetId::ASSET_SHADER_COMMON_UNIFORMS); + register_if_exists("math/sdf_shapes", AssetId::ASSET_SHADER_MATH_SDF_SHAPES); + register_if_exists("math/sdf_utils", AssetId::ASSET_SHADER_MATH_SDF_UTILS); + register_if_exists("render/shadows", AssetId::ASSET_SHADER_RENDER_SHADOWS); + register_if_exists("render/scene_query", AssetId::ASSET_SHADER_RENDER_SCENE_QUERY); + register_if_exists("render/lighting_utils", AssetId::ASSET_SHADER_RENDER_LIGHTING_UTILS); register_if_exists("sdf_primitives", AssetId::ASSET_SHADER_SDF_PRIMITIVES); diff --git a/src/tests/test_shader_composer.cc b/src/tests/test_shader_composer.cc index 16dabba..1dd8298 100644 --- a/src/tests/test_shader_composer.cc +++ b/src/tests/test_shader_composer.cc @@ -26,12 +26,12 @@ void test_composition() { std::string result = sc.Compose({"math", "util"}, main_code); // Verify order and presence - assert(result.find("Snippet: math") != std::string::npos); - assert(result.find("Snippet: util") != std::string::npos); + assert(result.find("Dependency: math") != std::string::npos); + assert(result.find("Dependency: util") != std::string::npos); assert(result.find("Main Code") != std::string::npos); - size_t pos_math = result.find("Snippet: math"); - size_t pos_util = result.find("Snippet: util"); + size_t pos_math = result.find("Dependency: math"); + size_t pos_util = result.find("Dependency: util"); size_t pos_main = result.find("Main Code"); assert(pos_math < pos_util); @@ -75,9 +75,56 @@ void test_asset_composition() { std::cout << "Asset-based composition logic verified." << std::endl; } +void test_recursive_composition() { + std::cout << "Testing Recursive Shader Composition..." << std::endl; + auto& sc = ShaderComposer::Get(); + + sc.RegisterSnippet("base", "fn base() {}"); + sc.RegisterSnippet("mid", "#include \"base\"\nfn mid() { base(); }"); + sc.RegisterSnippet("top", "#include \"mid\"\n#include \"base\"\nfn top() { mid(); base(); }"); + + std::string main_code = "#include \"top\"\nfn main() { top(); }"; + std::string result = sc.Compose({}, main_code); + + // Verify each is included exactly once despite multiple includes + size_t count_base = 0; + size_t pos = result.find("fn base()"); + while (pos != std::string::npos) { + count_base++; + pos = result.find("fn base()", pos + 1); + } + assert(count_base == 1); + + assert(result.find("Included: top") != std::string::npos); + assert(result.find("Included: mid") != std::string::npos); + assert(result.find("Included: base") != std::string::npos); + + std::cout << "Recursive composition logic verified." << std::endl; +} + +void test_renderer_composition() { + std::cout << "Testing Renderer Shader Composition..." << std::endl; + auto& sc = ShaderComposer::Get(); + + sc.RegisterSnippet("common_uniforms", "struct GlobalUniforms { view_proj: mat4x4 };"); + sc.RegisterSnippet("math/sdf_shapes", "fn sdSphere() {}"); + sc.RegisterSnippet("render/scene_query", "#include \"math/sdf_shapes\"\nfn map_scene() {}"); + + std::string main_code = "#include \"common_uniforms\"\n#include \"render/scene_query\"\nfn main() {}"; + std::string result = sc.Compose({}, main_code); + + assert(result.find("struct GlobalUniforms") != std::string::npos); + assert(result.find("fn sdSphere") != std::string::npos); + assert(result.find("fn map_scene") != std::string::npos); + + std::cout << "Renderer composition logic verified." << std::endl; +} + int main() { test_composition(); test_asset_composition(); + test_recursive_composition(); + test_renderer_composition(); std::cout << "--- ALL SHADER COMPOSER TESTS PASSED ---" << std::endl; return 0; } -- cgit v1.2.3