diff options
| author | skal <pascal.massimino@gmail.com> | 2026-02-12 12:17:59 +0100 |
|---|---|---|
| committer | skal <pascal.massimino@gmail.com> | 2026-02-12 12:17:59 +0100 |
| commit | ff4c1213636e66d4457a95cad12300c58e8d6781 (patch) | |
| tree | b47a9fc5c860c4eff39054b2ffc248ffbe19fa10 /scripts/train_cnn_v2_full.sh | |
| parent | eaf0bd855306e70ca03f2d6579b4d6551aff6482 (diff) | |
Refine training script output and validation
1. Loss printed at every epoch with \r (no scrolling)
2. Validation only on final epoch (not all checkpoints)
3. Process all input images (not just img_000.png)
Training output now shows live progress with single line update.
Diffstat (limited to 'scripts/train_cnn_v2_full.sh')
| -rwxr-xr-x | scripts/train_cnn_v2_full.sh | 45 |
1 files changed, 16 insertions, 29 deletions
diff --git a/scripts/train_cnn_v2_full.sh b/scripts/train_cnn_v2_full.sh index 383922d..7e5aeee 100755 --- a/scripts/train_cnn_v2_full.sh +++ b/scripts/train_cnn_v2_full.sh @@ -92,38 +92,25 @@ fi echo " → Build complete" echo "" -# Step 4: Visual assessment - process all checkpoints -echo "[4/4] Visual assessment of training progression..." +# Step 4: Visual assessment - process final checkpoint only +echo "[4/4] Visual assessment on all input images..." mkdir -p "$VALIDATION_DIR" -# Test first input image with checkpoints at intervals -TEST_IMAGE="$INPUT_DIR/img_000.png" -CHECKPOINT_INTERVAL=5 +echo " Processing final checkpoint: $FINAL_CHECKPOINT" -echo " Processing checkpoints (every ${CHECKPOINT_INTERVAL} epochs)..." +# Export final checkpoint shaders +python3 training/export_cnn_v2_weights.py "$FINAL_CHECKPOINT" \ + --output-weights workspaces/main/cnn_v2_weights.bin > /dev/null 2>&1 -for checkpoint in "$CHECKPOINT_DIR"/checkpoint_epoch_*.pth; do - epoch=$(echo "$checkpoint" | grep -o 'epoch_[0-9]*' | cut -d'_' -f2) - - # Only process checkpoints at intervals - if [ $((epoch % CHECKPOINT_INTERVAL)) -eq 0 ] || [ "$epoch" -eq "$EPOCHS" ]; then - echo " Epoch $epoch..." - - # Export shaders for this checkpoint - python3 training/export_cnn_v2_shader.py "$checkpoint" \ - --output-dir workspaces/main/shaders > /dev/null 2>&1 - - # Rebuild - cmake --build build -j4 --target cnn_test > /dev/null 2>&1 - - # Process test image - build/cnn_test "$TEST_IMAGE" "$VALIDATION_DIR/epoch_${epoch}_output.png" 2>/dev/null - fi -done +# Rebuild with final weights +cmake --build build -j4 --target cnn_test > /dev/null 2>&1 -# Restore final checkpoint shaders -python3 training/export_cnn_v2_shader.py "$FINAL_CHECKPOINT" \ - --output-dir workspaces/main/shaders > /dev/null 2>&1 +# Process all input images +for input_image in "$INPUT_DIR"/*.png; do + basename=$(basename "$input_image" .png) + echo " Processing $basename..." + build/cnn_test "$input_image" "$VALIDATION_DIR/${basename}_output.png" 2>/dev/null +done cmake --build build -j4 --target demo64k > /dev/null 2>&1 @@ -132,8 +119,8 @@ echo "=== Training Pipeline Complete ===" echo "" echo "Results:" echo " - Checkpoints: $CHECKPOINT_DIR" -echo " - Visual progression: $VALIDATION_DIR" -echo " - Final shaders: workspaces/main/shaders/cnn_v2_layer_*.wgsl" +echo " - Validation outputs: $VALIDATION_DIR" +echo " - Final weights: workspaces/main/cnn_v2_weights.bin" echo "" echo "Opening results directory..." open "$VALIDATION_DIR" 2>/dev/null || xdg-open "$VALIDATION_DIR" 2>/dev/null || true |
