diff options
Diffstat (limited to 'cnn_v3/training')
29 files changed, 37 insertions, 17 deletions
diff --git a/cnn_v3/training/cnn_v3_utils.py b/cnn_v3/training/cnn_v3_utils.py index 5a3d56c..bef4091 100644 --- a/cnn_v3/training/cnn_v3_utils.py +++ b/cnn_v3/training/cnn_v3_utils.py @@ -11,7 +11,7 @@ Imported by train_cnn_v3.py and export_cnn_v3_weights.py. [9-11] prev.rgb f32 (zero during training) [12-14] mip1.rgb pyrdown(albedo) [15-17] mip2.rgb pyrdown(mip1) - [18] shadow f32 [0,1] + [18] dif f32 [0,1] max(0,dot(normal,KEY_LIGHT))*shadow [19] transp f32 [0,1] Sample directory layout (per sample_xxx/): @@ -48,10 +48,11 @@ from torch.utils.data import Dataset N_FEATURES = 20 GEOMETRIC_CHANNELS = [3, 4, 5, 6, 7] # normal.xy, depth, depth_grad.xy -CONTEXT_CHANNELS = [8, 18, 19] # mat_id, shadow, transp +CONTEXT_CHANNELS = [8, 18, 19] # mat_id, dif, transp TEMPORAL_CHANNELS = [9, 10, 11] # prev.rgb -_LUMA = np.array([0.2126, 0.7152, 0.0722], dtype=np.float32) # BT.709 +_LUMA = np.array([0.2126, 0.7152, 0.0722], dtype=np.float32) # BT.709 +_KEY_LIGHT = np.array([0.408, 0.816, 0.408 ], dtype=np.float32) # normalize(1,2,1) # --------------------------------------------------------------------------- # Image I/O @@ -102,6 +103,21 @@ def depth_gradient(depth: np.ndarray) -> np.ndarray: return np.stack([dzdx, dzdy], axis=-1) +def oct_decode(enc: np.ndarray) -> np.ndarray: + """Decode oct-encoded normals (H,W,2) in [0,1] → (H,W,3) unit normals.""" + f = enc * 2.0 - 1.0 # [0,1] → [-1,1] + z = 1.0 - np.abs(f[..., :1]) - np.abs(f[..., 1:2]) + n = np.concatenate([f, z], axis=-1) + neg = n[..., 2:3] < 0.0 + n = np.concatenate([ + np.where(neg, (1.0 - np.abs(f[..., 1:2])) * np.sign(f[..., :1]), n[..., :1]), + np.where(neg, (1.0 - np.abs(f[..., :1])) * np.sign(f[..., 1:2]), n[..., 1:2]), + n[..., 2:3], + ], axis=-1) + length = np.linalg.norm(n, axis=-1, keepdims=True) + return n / np.maximum(length, 1e-8) + + def _upsample_nearest(a: np.ndarray, h: int, w: int) -> np.ndarray: """Nearest-neighbour upsample (H,W,C) f32 to (h,w,C) — pure numpy, no precision loss.""" sh, sw = a.shape[:2] @@ -117,25 +133,29 @@ def assemble_features(albedo: np.ndarray, normal: np.ndarray, prev set to zero (no temporal history during training). mip1/mip2 computed from albedo. depth_grad computed via finite diff. + dif (ch18) = max(0, dot(oct_decode(normal), KEY_LIGHT)) * shadow. """ h, w = albedo.shape[:2] - mip1 = _upsample_nearest(pyrdown(albedo), h, w) - mip2 = _upsample_nearest(pyrdown(pyrdown(albedo)), h, w) - dgrad = depth_gradient(depth) - prev = np.zeros((h, w, 3), dtype=np.float32) + mip1 = _upsample_nearest(pyrdown(albedo), h, w) + mip2 = _upsample_nearest(pyrdown(pyrdown(albedo)), h, w) + dgrad = depth_gradient(depth) + prev = np.zeros((h, w, 3), dtype=np.float32) + nor3 = oct_decode(normal) + diffuse = np.maximum(0.0, (nor3 * _KEY_LIGHT).sum(-1)) + dif = diffuse * shadow return np.concatenate([ - albedo, # [0-2] albedo.rgb - normal, # [3-4] normal.xy - depth[..., None], # [5] depth - dgrad, # [6-7] depth_grad.xy - matid[..., None], # [8] mat_id - prev, # [9-11] prev.rgb - mip1, # [12-14] mip1.rgb - mip2, # [15-17] mip2.rgb - shadow[..., None], # [18] shadow - transp[..., None], # [19] transp + albedo, # [0-2] albedo.rgb + normal, # [3-4] normal.xy + depth[..., None], # [5] depth + dgrad, # [6-7] depth_grad.xy + matid[..., None], # [8] mat_id + prev, # [9-11] prev.rgb + mip1, # [12-14] mip1.rgb + mip2, # [15-17] mip2.rgb + dif[..., None], # [18] dif = diffuse * shadow + transp[..., None],# [19] transp ], axis=-1).astype(np.float32) diff --git a/cnn_v3/training/dataset/full/0001/albedo.png b/cnn_v3/training/dataset/full/0001/albedo.png Binary files differnew file mode 100644 index 0000000..8f64b38 --- /dev/null +++ b/cnn_v3/training/dataset/full/0001/albedo.png diff --git a/cnn_v3/training/dataset/full/0001/depth.png b/cnn_v3/training/dataset/full/0001/depth.png Binary files differnew file mode 100644 index 0000000..c58fcd9 --- /dev/null +++ b/cnn_v3/training/dataset/full/0001/depth.png diff --git a/cnn_v3/training/dataset/full/0001/matid.png b/cnn_v3/training/dataset/full/0001/matid.png Binary files differnew file mode 100644 index 0000000..b4fa98f --- /dev/null +++ b/cnn_v3/training/dataset/full/0001/matid.png diff --git a/cnn_v3/training/dataset/full/0001/normal.png b/cnn_v3/training/dataset/full/0001/normal.png Binary files differnew file mode 100644 index 0000000..62f26e3 --- /dev/null +++ b/cnn_v3/training/dataset/full/0001/normal.png diff --git a/cnn_v3/training/dataset/full/0001/shadow.png b/cnn_v3/training/dataset/full/0001/shadow.png Binary files differnew file mode 100644 index 0000000..0471e7f --- /dev/null +++ b/cnn_v3/training/dataset/full/0001/shadow.png diff --git a/cnn_v3/training/dataset/full/0001/target.png b/cnn_v3/training/dataset/full/0001/target.png Binary files differnew file mode 100644 index 0000000..587d54a --- /dev/null +++ b/cnn_v3/training/dataset/full/0001/target.png diff --git a/cnn_v3/training/dataset/full/0001/transp.png b/cnn_v3/training/dataset/full/0001/transp.png Binary files differnew file mode 100644 index 0000000..b4fa98f --- /dev/null +++ b/cnn_v3/training/dataset/full/0001/transp.png diff --git a/cnn_v3/training/dataset/full/0002/albedo.png b/cnn_v3/training/dataset/full/0002/albedo.png Binary files differnew file mode 100644 index 0000000..8f64b38 --- /dev/null +++ b/cnn_v3/training/dataset/full/0002/albedo.png diff --git a/cnn_v3/training/dataset/full/0002/depth.png b/cnn_v3/training/dataset/full/0002/depth.png Binary files differnew file mode 100644 index 0000000..c58fcd9 --- /dev/null +++ b/cnn_v3/training/dataset/full/0002/depth.png diff --git a/cnn_v3/training/dataset/full/0002/matid.png b/cnn_v3/training/dataset/full/0002/matid.png Binary files differnew file mode 100644 index 0000000..b4fa98f --- /dev/null +++ b/cnn_v3/training/dataset/full/0002/matid.png diff --git a/cnn_v3/training/dataset/full/0002/normal.png b/cnn_v3/training/dataset/full/0002/normal.png Binary files differnew file mode 100644 index 0000000..62f26e3 --- /dev/null +++ b/cnn_v3/training/dataset/full/0002/normal.png diff --git a/cnn_v3/training/dataset/full/0002/shadow.png b/cnn_v3/training/dataset/full/0002/shadow.png Binary files differnew file mode 100644 index 0000000..0471e7f --- /dev/null +++ b/cnn_v3/training/dataset/full/0002/shadow.png diff --git a/cnn_v3/training/dataset/full/0002/target.png b/cnn_v3/training/dataset/full/0002/target.png Binary files differnew file mode 100644 index 0000000..587d54a --- /dev/null +++ b/cnn_v3/training/dataset/full/0002/target.png diff --git a/cnn_v3/training/dataset/full/0002/transp.png b/cnn_v3/training/dataset/full/0002/transp.png Binary files differnew file mode 100644 index 0000000..b4fa98f --- /dev/null +++ b/cnn_v3/training/dataset/full/0002/transp.png diff --git a/cnn_v3/training/dataset/full/0003/albedo.png b/cnn_v3/training/dataset/full/0003/albedo.png Binary files differnew file mode 100644 index 0000000..8f64b38 --- /dev/null +++ b/cnn_v3/training/dataset/full/0003/albedo.png diff --git a/cnn_v3/training/dataset/full/0003/depth.png b/cnn_v3/training/dataset/full/0003/depth.png Binary files differnew file mode 100644 index 0000000..c58fcd9 --- /dev/null +++ b/cnn_v3/training/dataset/full/0003/depth.png diff --git a/cnn_v3/training/dataset/full/0003/matid.png b/cnn_v3/training/dataset/full/0003/matid.png Binary files differnew file mode 100644 index 0000000..b4fa98f --- /dev/null +++ b/cnn_v3/training/dataset/full/0003/matid.png diff --git a/cnn_v3/training/dataset/full/0003/normal.png b/cnn_v3/training/dataset/full/0003/normal.png Binary files differnew file mode 100644 index 0000000..62f26e3 --- /dev/null +++ b/cnn_v3/training/dataset/full/0003/normal.png diff --git a/cnn_v3/training/dataset/full/0003/shadow.png b/cnn_v3/training/dataset/full/0003/shadow.png Binary files differnew file mode 100644 index 0000000..0471e7f --- /dev/null +++ b/cnn_v3/training/dataset/full/0003/shadow.png diff --git a/cnn_v3/training/dataset/full/0003/target.png b/cnn_v3/training/dataset/full/0003/target.png Binary files differnew file mode 100644 index 0000000..587d54a --- /dev/null +++ b/cnn_v3/training/dataset/full/0003/target.png diff --git a/cnn_v3/training/dataset/full/0003/transp.png b/cnn_v3/training/dataset/full/0003/transp.png Binary files differnew file mode 100644 index 0000000..b4fa98f --- /dev/null +++ b/cnn_v3/training/dataset/full/0003/transp.png diff --git a/cnn_v3/training/dataset/full/0004/albedo.png b/cnn_v3/training/dataset/full/0004/albedo.png Binary files differnew file mode 100644 index 0000000..8f64b38 --- /dev/null +++ b/cnn_v3/training/dataset/full/0004/albedo.png diff --git a/cnn_v3/training/dataset/full/0004/depth.png b/cnn_v3/training/dataset/full/0004/depth.png Binary files differnew file mode 100644 index 0000000..c58fcd9 --- /dev/null +++ b/cnn_v3/training/dataset/full/0004/depth.png diff --git a/cnn_v3/training/dataset/full/0004/matid.png b/cnn_v3/training/dataset/full/0004/matid.png Binary files differnew file mode 100644 index 0000000..b4fa98f --- /dev/null +++ b/cnn_v3/training/dataset/full/0004/matid.png diff --git a/cnn_v3/training/dataset/full/0004/normal.png b/cnn_v3/training/dataset/full/0004/normal.png Binary files differnew file mode 100644 index 0000000..62f26e3 --- /dev/null +++ b/cnn_v3/training/dataset/full/0004/normal.png diff --git a/cnn_v3/training/dataset/full/0004/shadow.png b/cnn_v3/training/dataset/full/0004/shadow.png Binary files differnew file mode 100644 index 0000000..0471e7f --- /dev/null +++ b/cnn_v3/training/dataset/full/0004/shadow.png diff --git a/cnn_v3/training/dataset/full/0004/target.png b/cnn_v3/training/dataset/full/0004/target.png Binary files differnew file mode 100644 index 0000000..587d54a --- /dev/null +++ b/cnn_v3/training/dataset/full/0004/target.png diff --git a/cnn_v3/training/dataset/full/0004/transp.png b/cnn_v3/training/dataset/full/0004/transp.png Binary files differnew file mode 100644 index 0000000..b4fa98f --- /dev/null +++ b/cnn_v3/training/dataset/full/0004/transp.png |
