feat: Add checkpointing support to CNN training script

- Save checkpoints every N epochs (--checkpoint-every)
- Resume from checkpoint (--resume)
- Store model, optimizer, epoch, loss, and architecture info
- Auto-create checkpoint directory

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

1 files changed, 158 insertions, 0 deletions

diff --git a/training/workspaces/main/shaders/cnn/cnn_weights_generated.wgsl b/training/workspaces/main/shaders/cnn/cnn_weights_generated.wgsl
new file mode 100644
index 0000000..dae81df
--- /dev/null
+++ b/training/workspaces/main/shaders/cnn/cnn_weights_generated.wgsl
@@ -0,0 +1,158 @@
+// Auto-generated CNN weights
+// DO NOT EDIT - Generated by train_cnn.py
+
+const rgba_weights_layer0: array<mat4x4<f32>, 9> = array(
+  mat4x4<f32>(
+    -0.019696, -0.045138, -0.059372,
+    0.113637, -0.026176, -0.204699,
+    -0.147723, -0.124720, -0.133641,
+  ),
+  mat4x4<f32>(
+    -0.011820, -0.110039, -0.019111,
+    0.102596, -0.053469, -0.090972,
+    -0.106286, 0.062616, -0.211309,
+  ),
+  mat4x4<f32>(
+    0.169672, -0.188668, 0.097992,
+    -0.048049, 0.035012, -0.028287,
+    0.041841, 0.113846, 0.092006,
+  ),
+  mat4x4<f32>(
+    0.084688, -0.173117, -0.130135,
+    0.125052, 0.070060, -0.072493,
+    -0.081996, -0.041021, -0.200688,
+  ),
+  mat4x4<f32>(
+    0.180550, 0.018555, -0.092889,
+    0.105823, 0.109215, 0.042989,
+    -0.116116, 0.115354, 0.044726,
+  ),
+  mat4x4<f32>(
+    0.069597, -0.156086, -0.116919,
+    0.003641, -0.033090, 0.077686,
+    -0.090117, 0.047527, 0.093449,
+  ),
+  mat4x4<f32>(
+    -0.007961, -0.201232, -0.094087,
+    0.041521, -0.001265, -0.164458,
+    -0.063295, -0.177367, 0.120887,
+  ),
+  mat4x4<f32>(
+    0.005358, -0.153663, 0.234817,
+    0.094452, -0.030598, -0.159715,
+    -0.025096, 0.010606, -0.151786,
+  ),
+  mat4x4<f32>(
+    0.035922, 0.039006, -0.073426,
+    0.234309, 0.042990, -0.074330,
+    0.129497, -0.084083, -0.165691,
+  )
+);
+
+const coord_weights_layer0 = mat2x4<f32>(
+  0.156995, -0.026005, 0.159550,
+  0.112678, -0.021301, 0.106653
+);
+
+const bias_layer0 = vec4<f32>(0.149566, -0.002723, 0.142744);
+
+const weights_layer1: array<mat4x4<f32>, 9> = array(
+  mat4x4<f32>(
+    0.198730, -0.060590, -0.126001,
+    0.018094, 0.099855, 0.043531,
+    -0.048028, 0.024975, -0.055560,
+  ),
+  mat4x4<f32>(
+    0.093012, -0.056168, 0.075685,
+    -0.104572, 0.202161, 0.093453,
+    0.008470, 0.190414, -0.121853,
+  ),
+  mat4x4<f32>(
+    0.157523, -0.278521, 0.267972,
+    0.226318, 0.108021, -0.020615,
+    0.116906, 0.094663, 0.103058,
+  ),
+  mat4x4<f32>(
+    0.184815, -0.167385, -0.081513,
+    0.167595, 0.147724, -0.034069,
+    0.109272, 0.149283, 0.022741,
+  ),
+  mat4x4<f32>(
+    -0.133319, 0.069405, 0.028862,
+    -0.044914, -0.121720, 0.074758,
+    0.150973, 0.086887, 0.193997,
+  ),
+  mat4x4<f32>(
+    0.123384, -0.157817, -0.053264,
+    0.216874, 0.024062, 0.227470,
+    0.092232, 0.156942, 0.098989,
+  ),
+  mat4x4<f32>(
+    -0.074328, -0.265180, 0.065633,
+    0.033679, 0.175748, 0.178567,
+    0.168913, 0.192317, -0.015507,
+  ),
+  mat4x4<f32>(
+    -0.103567, -0.081663, 0.239707,
+    0.020591, 0.031346, 0.089577,
+    -0.040636, 0.061481, 0.215428,
+  ),
+  mat4x4<f32>(
+    0.103399, -0.291323, 0.220388,
+    0.163876, 0.106383, 0.175615,
+    0.050511, 0.210950, -0.143280,
+  )
+);
+
+const bias_layer1 = vec4<f32>(0.273340, 0.183151, 0.057200);
+
+const weights_layer2: array<mat4x4<f32>, 9> = array(
+  mat4x4<f32>(
+    0.170688, -0.158379, -0.073057,
+    -0.213429, -0.075772, -0.117451,
+    -0.265536, -0.066896, 0.185188,
+  ),
+  mat4x4<f32>(
+    0.061069, -0.267237, -0.057030,
+    -0.112682, -0.001723, 0.020779,
+    -0.158726, -0.027319, -0.133134,
+  ),
+  mat4x4<f32>(
+    -0.036597, 0.000282, -0.286058,
+    -0.056992, 0.129227, 0.037650,
+    -0.305341, -0.082011, 0.155333,
+  ),
+  mat4x4<f32>(
+    0.146811, 0.086471, -0.092652,
+    -0.083987, -0.164501, 0.005801,
+    -0.108568, 0.079618, 0.011061,
+  ),
+  mat4x4<f32>(
+    0.008716, -0.174373, 0.038516,
+    -0.263207, -0.201249, -0.106428,
+    -0.321199, 0.139540, -0.069047,
+  ),
+  mat4x4<f32>(
+    -0.099231, -0.037154, -0.189117,
+    0.014380, 0.102996, 0.068944,
+    -0.011073, 0.175106, 0.019059,
+  ),
+  mat4x4<f32>(
+    -0.170030, -0.077528, -0.038504,
+    0.042379, -0.198288, 0.008895,
+    -0.144090, -0.129658, 0.215823,
+  ),
+  mat4x4<f32>(
+    -0.082481, -0.160808, -0.279220,
+    -0.029358, 0.021159, -0.037080,
+    -0.194849, -0.013461, 0.057026,
+  ),
+  mat4x4<f32>(
+    -0.063711, -0.198759, -0.037847,
+    -0.049292, -0.222896, -0.067384,
+    -0.167766, -0.090320, 0.106986,
+  )
+);
+
+const bias_layer2 = vec4<f32>(0.021260, -0.056985, 0.000823);
+