summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorskal <pascal.massimino@gmail.com>2026-03-25 10:48:08 +0100
committerskal <pascal.massimino@gmail.com>2026-03-25 10:48:08 +0100
commit3b2c9c04d3380c2bfd87e2fea1ba92574590766b (patch)
tree79a733f4ef7c5904a5aead57708d846332f2a132
parentce6e5b99f26e4e7c69a3cacf360bd0d492de928c (diff)
feat(cnn_v3/training): load prev.png when available; document web tool prev gapHEADmain
- assemble_features() accepts optional prev ndarray (None → zeros) - _load_sample() loads prev.png if present, else None - __getitem__ slices/resizes prev alongside other channels - TODO.md: note that cnn_v3/tools/shaders.js hardcodes prev=0 in both pack shaders while C++ gbuf_pack.wgsl reads a real prev_cnn texture handoff(Gemini): prev.png now used in training when present; web tool gap documented in TODO.md
-rw-r--r--TODO.md4
-rw-r--r--cnn_v3/training/cnn_v3_utils.py20
2 files changed, 18 insertions, 6 deletions
diff --git a/TODO.md b/TODO.md
index 0184f0a..c5b71e9 100644
--- a/TODO.md
+++ b/TODO.md
@@ -76,6 +76,10 @@ Ongoing shader code hygiene for granular, reusable snippets.
**Pending (lower priority):**
- [ ] GBufferEffect: Pass 3 transparency (transp=0 placeholder)
- [ ] GBufferEffect: `resize()` support
+- [ ] Web tool (`cnn_v3/tools/shaders.js`): `prev_cnn` always zero in both pack shaders
+ (`FULL_PACK_SHADER` line ~313 and simple pack line ~39 hardcode `prev=0`).
+ C++ `gbuf_pack.wgsl` reads a real `prev_cnn` texture (binding 6).
+ Fix: add a `prev` texture binding to both JS pack shaders and wire it up in `tester.js`.
## Future: CNN v3 "2D Mode" (G-buffer-free)
diff --git a/cnn_v3/training/cnn_v3_utils.py b/cnn_v3/training/cnn_v3_utils.py
index 50707a2..68c0798 100644
--- a/cnn_v3/training/cnn_v3_utils.py
+++ b/cnn_v3/training/cnn_v3_utils.py
@@ -128,10 +128,11 @@ def _upsample_nearest(a: np.ndarray, h: int, w: int) -> np.ndarray:
def assemble_features(albedo: np.ndarray, normal: np.ndarray,
depth: np.ndarray, matid: np.ndarray,
- shadow: np.ndarray, transp: np.ndarray) -> np.ndarray:
+ shadow: np.ndarray, transp: np.ndarray,
+ prev: np.ndarray | None = None) -> np.ndarray:
"""Build (H,W,20) f32 feature tensor.
- prev set to zero (no temporal history during training).
+ prev: (H,W,3) f32 [0,1] previous frame RGB, or None → zeros.
mip1/mip2 computed from albedo. depth_grad computed via finite diff.
dif (ch18) = max(0, dot(oct_decode(normal), KEY_LIGHT)) * shadow.
"""
@@ -140,7 +141,8 @@ def assemble_features(albedo: np.ndarray, normal: np.ndarray,
mip1 = _upsample_nearest(pyrdown(albedo), h, w)
mip2 = _upsample_nearest(pyrdown(pyrdown(albedo)), h, w)
dgrad = depth_gradient(depth)
- prev = np.zeros((h, w, 3), dtype=np.float32)
+ if prev is None:
+ prev = np.zeros((h, w, 3), dtype=np.float32)
nor3 = oct_decode(normal)
diffuse = np.maximum(0.0, (nor3 * _KEY_LIGHT).sum(-1))
dif = diffuse * shadow
@@ -348,11 +350,13 @@ class CNNv3Dataset(Dataset):
shadow = load_gray(sd / 'shadow.png')
transp = load_gray(sd / 'transp.png')
h, w = albedo.shape[:2]
+ prev_path = sd / 'prev.png'
+ prev = load_rgb(prev_path) if prev_path.exists() else None
target_img = Image.open(sd / 'target.png').convert('RGBA')
if target_img.size != (w, h):
target_img = target_img.resize((w, h), Image.LANCZOS)
target = np.asarray(target_img, dtype=np.float32) / 255.0
- return albedo, normal, depth, matid, shadow, transp, target
+ return albedo, normal, depth, matid, shadow, transp, prev, target
def __getitem__(self, idx):
if self.full_image:
@@ -360,7 +364,7 @@ class CNNv3Dataset(Dataset):
else:
sample_idx = idx // self.patches_per_image
- albedo, normal, depth, matid, shadow, transp, target = self._cache[sample_idx]
+ albedo, normal, depth, matid, shadow, transp, prev, target = self._cache[sample_idx]
h, w = albedo.shape[:2]
if self.full_image:
@@ -382,6 +386,8 @@ class CNNv3Dataset(Dataset):
matid = _resize_gray(matid)
shadow = _resize_gray(shadow)
transp = _resize_gray(transp)
+ if prev is not None:
+ prev = _resize_img(prev)
target = _resize_img(target)
else:
ps = self.patch_size
@@ -398,6 +404,8 @@ class CNNv3Dataset(Dataset):
matid = matid[sl]
shadow = shadow[sl]
transp = transp[sl]
+ if prev is not None:
+ prev = prev[sl]
# Apply cached target offset (if search was enabled at init).
if self._target_offsets:
@@ -408,7 +416,7 @@ class CNNv3Dataset(Dataset):
else:
target = target[sl]
- feat = assemble_features(albedo, normal, depth, matid, shadow, transp)
+ feat = assemble_features(albedo, normal, depth, matid, shadow, transp, prev)
if self.augment:
feat = apply_channel_dropout(feat,