path: root/scripts/train_cnn_v2_full.sh

#!/bin/bash
# Complete CNN v2 Training Pipeline
# Train → Export → Build → Validate
# Usage: ./train_cnn_v2_full.sh [OPTIONS]
#
# OPTIONS:
#   (none)                    Run complete pipeline: train → export → build → validate
#   --validate                Validate only (skip training, use existing weights)
#   --validate CHECKPOINT     Validate with specific checkpoint file
#   --export-only CHECKPOINT  Export weights only (skip training, build, validation)
#   --mip-level LEVEL         Mip level for p0-p3 features: 0=original, 1=half, 2=quarter, 3=eighth (default: 0)
#   --help                    Show this help message
#
# Examples:
#   ./train_cnn_v2_full.sh
#   ./train_cnn_v2_full.sh --validate
#   ./train_cnn_v2_full.sh --validate checkpoints/checkpoint_epoch_50.pth
#   ./train_cnn_v2_full.sh --export-only checkpoints/checkpoint_epoch_100.pth
#   ./train_cnn_v2_full.sh --mip-level 1

set -e

PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$PROJECT_ROOT"

# Parse arguments
VALIDATE_ONLY=false
VALIDATE_CHECKPOINT=""
EXPORT_ONLY=false
EXPORT_CHECKPOINT=""
MIP_LEVEL=0

if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
    head -21 "$0" | grep "^#" | grep -v "^#!/" | sed 's/^# *//'
    exit 0
fi

# Parse all arguments
while [[ $# -gt 0 ]]; do
    case "$1" in
        --export-only)
            EXPORT_ONLY=true
            if [ -z "$2" ]; then
                echo "Error: --export-only requires a checkpoint file argument"
                exit 1
            fi
            EXPORT_CHECKPOINT="$2"
            shift 2
            ;;
        --validate)
            VALIDATE_ONLY=true
            if [ -n "$2" ] && [[ ! "$2" =~ ^-- ]]; then
                VALIDATE_CHECKPOINT="$2"
                shift 2
            else
                shift
            fi
            ;;
        --mip-level)
            if [ -z "$2" ]; then
                echo "Error: --mip-level requires a level argument (0-3)"
                exit 1
            fi
            MIP_LEVEL="$2"
            shift 2
            ;;
        *)
            echo "Unknown option: $1"
            exit 1
            ;;
    esac
done

# Configuration
INPUT_DIR="training/input"
TARGET_DIR="training/target_2"
CHECKPOINT_DIR="checkpoints"
VALIDATION_DIR="validation_results"
EPOCHS=200
CHECKPOINT_EVERY=50
BATCH_SIZE=16

# Patch-based training (default)
PATCH_SIZE=8
PATCHES_PER_IMAGE=256
DETECTOR="harris"

# Full-image training (alternative - uncomment to use)
# FULL_IMAGE="--full-image"
# IMAGE_SIZE=256

KERNEL_SIZES="3,3,3"  # Comma-separated per-layer kernel sizes
NUM_LAYERS=3
# MIP_LEVEL set via --mip-level argument (default: 0)

# Handle export-only mode
if [ "$EXPORT_ONLY" = true ]; then
    echo "=== CNN v2 Export Weights Only ==="
    echo "Checkpoint: $EXPORT_CHECKPOINT"
    echo ""

    if [ ! -f "$EXPORT_CHECKPOINT" ]; then
        echo "Error: Checkpoint file not found: $EXPORT_CHECKPOINT"
        exit 1
    fi

    python3 training/export_cnn_v2_weights.py "$EXPORT_CHECKPOINT" \
      --output-weights workspaces/main/weights/cnn_v2_weights.bin

    if [ $? -ne 0 ]; then
        echo "Error: Export failed"
        exit 1
    fi

    echo ""
    echo "=== Export Complete ==="
    echo "Output: workspaces/main/weights/cnn_v2_weights.bin"
    exit 0
fi

if [ "$VALIDATE_ONLY" = true ]; then
    echo "=== CNN v2 Validation Only ==="
    echo "Skipping training, using existing weights"
    echo ""
else
    echo "=== CNN v2 Complete Training Pipeline ==="
    echo "Input: $INPUT_DIR"
    echo "Target: $TARGET_DIR"
    echo "Epochs: $EPOCHS"
    echo "Checkpoint interval: $CHECKPOINT_EVERY"
    echo "Mip level: $MIP_LEVEL (p0-p3 features)"
    echo ""
fi

if [ "$VALIDATE_ONLY" = false ]; then
    # Step 1: Train model
    echo "[1/4] Training CNN v2 model..."
python3 training/train_cnn_v2.py \
  --input "$INPUT_DIR" \
  --target "$TARGET_DIR" \
  --patch-size $PATCH_SIZE \
  --patches-per-image $PATCHES_PER_IMAGE \
  --detector $DETECTOR \
  --kernel-sizes $KERNEL_SIZES \
  --num-layers $NUM_LAYERS \
  --mip-level $MIP_LEVEL \
  --epochs $EPOCHS \
  --batch-size $BATCH_SIZE \
  --checkpoint-dir "$CHECKPOINT_DIR" \
  --checkpoint-every $CHECKPOINT_EVERY \
  $FULL_IMAGE

if [ $? -ne 0 ]; then
    echo "Error: Training failed"
    exit 1
fi

echo ""
echo "Training complete!"
echo ""

# Step 2: Export final checkpoint to shaders
FINAL_CHECKPOINT="$CHECKPOINT_DIR/checkpoint_epoch_${EPOCHS}.pth"

if [ ! -f "$FINAL_CHECKPOINT" ]; then
    echo "Warning: Final checkpoint not found, using latest available..."
    FINAL_CHECKPOINT=$(ls -t "$CHECKPOINT_DIR"/checkpoint_epoch_*.pth | head -1)
fi

echo "[2/4] Exporting final checkpoint to binary weights..."
echo "Checkpoint: $FINAL_CHECKPOINT"
python3 training/export_cnn_v2_weights.py "$FINAL_CHECKPOINT" \
  --output-weights workspaces/main/weights/cnn_v2_weights.bin

if [ $? -ne 0 ]; then
    echo "Error: Shader export failed"
    exit 1
fi

echo ""
fi  # End of training/export section

# Determine which checkpoint to use
if [ "$VALIDATE_ONLY" = true ]; then
    if [ -n "$VALIDATE_CHECKPOINT" ]; then
        FINAL_CHECKPOINT="$VALIDATE_CHECKPOINT"
    else
        # Use latest checkpoint
        FINAL_CHECKPOINT=$(ls -t "$CHECKPOINT_DIR"/checkpoint_epoch_*.pth | head -1)
    fi
    echo "Using checkpoint: $FINAL_CHECKPOINT"
    echo ""
fi

# Step 3: Rebuild with new shaders
if [ "$VALIDATE_ONLY" = false ]; then
    echo "[3/4] Rebuilding demo with new shaders..."
    cmake --build build -j4 --target demo64k > /dev/null 2>&1

    if [ $? -ne 0 ]; then
        echo "Error: Build failed"
        exit 1
    fi

    echo "  → Build complete"
    echo ""
fi

# Step 4: Visual assessment - process final checkpoint only
if [ "$VALIDATE_ONLY" = true ]; then
    echo "Validation on all input images (using existing weights)..."
else
    echo "[4/4] Visual assessment on all input images..."
fi

mkdir -p "$VALIDATION_DIR"
echo "  Using checkpoint: $FINAL_CHECKPOINT"

# Export weights for validation mode (already exported in step 2 for training mode)
if [ "$VALIDATE_ONLY" = true ]; then
    python3 training/export_cnn_v2_weights.py "$FINAL_CHECKPOINT" \
      --output-weights workspaces/main/weights/cnn_v2_weights.bin > /dev/null 2>&1
fi

# Build cnn_test
cmake --build build -j4 --target cnn_test > /dev/null 2>&1

# Process all input images
for input_image in "$INPUT_DIR"/*.png; do
    basename=$(basename "$input_image" .png)
    echo "    Processing $basename..."
    build/cnn_test "$input_image" "$VALIDATION_DIR/${basename}_output.png" 2>/dev/null
done

# Build demo only if not in validate mode
if [ "$VALIDATE_ONLY" = false ]; then
    cmake --build build -j4 --target demo64k > /dev/null 2>&1
fi

echo ""
if [ "$VALIDATE_ONLY" = true ]; then
    echo "=== Validation Complete ==="
else
    echo "=== Training Pipeline Complete ==="
fi
echo ""
echo "Results:"
if [ "$VALIDATE_ONLY" = false ]; then
    echo "  - Checkpoints: $CHECKPOINT_DIR"
    echo "  - Final weights: workspaces/main/weights/cnn_v2_weights.bin"
fi
echo "  - Validation outputs: $VALIDATION_DIR"
echo ""
echo "Opening results directory..."
open "$VALIDATION_DIR" 2>/dev/null || xdg-open "$VALIDATION_DIR" 2>/dev/null || true

if [ "$VALIDATE_ONLY" = false ]; then
    echo ""
    echo "Run demo to see final result:"
    echo "  ./build/demo64k"
fi