From d3d72f7b70e8f7d1f1ec0fb3030c447a8f876d8f Mon Sep 17 00:00:00 2001 From: skal Date: Fri, 13 Feb 2026 15:59:54 +0100 Subject: CNN v2 test tool: Add video playback support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Features: - Video file support (MP4, WebM, etc.) via drag-and-drop - Play/Pause button with non-realtime playback (drops frames if CNN slow) - Frame-by-frame navigation (◄/► step buttons) - Unified image/video processing through same CNN pipeline - Audio muted (video frames only) Optimizations: - Layer visualization updates only on pause/seek (~5-10ms saved per frame) Architecture: - copyExternalImageToTexture() works with both ImageBitmap and HTMLVideoElement - Video loading: wait for metadata → seek to frame 0 → wait for readyState≥2 (decoded) - Playback loop: requestAnimationFrame with isProcessing guard prevents overlapping inference - Controls always visible, disabled for images Co-Authored-By: Claude Sonnet 4.5 --- tools/cnn_v2_test/index.html | 188 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 166 insertions(+), 22 deletions(-) (limited to 'tools/cnn_v2_test/index.html') diff --git a/tools/cnn_v2_test/index.html b/tools/cnn_v2_test/index.html index cc045f1..95ec788 100644 --- a/tools/cnn_v2_test/index.html +++ b/tools/cnn_v2_test/index.html @@ -14,9 +14,19 @@ - weights.layers[] array: Layer 0 = weights.layers[0], Layer 1 = weights.layers[1] Features: + - Input: PNG images or video files (MP4, WebM, etc.) + - Video playback: Play/Pause, frame-by-frame navigation (◄/► buttons) + - Video mode: Non-realtime processing (drops frames if CNN slower than playback) - Side panel: .bin metadata display, weight statistics per layer - Layer inspection: 4-channel grayscale split, intermediate layer visualization - View modes: CNN output, original, diff (×10) + - Optimization: Layer viz updates only on pause/seek during video playback + + WGSL Shader Reuse: + - CNN_SHADER (inference), STATIC_SHADER, LAYER_VIZ_SHADER are inline for single-file deployment + - Can extract to .wgsl files for: better IDE support, testing, cross-tool reuse + - Tradeoff: extraction needs fetch() or build step, breaks single-file portability + - C++ sync: manual (WGSL ≠ GLSL) but logic identical --> @@ -67,6 +77,20 @@ background: #1a1a1a; border-radius: 4px; } + button { + background: #1a1a1a; + border: 1px solid #404040; + color: #e0e0e0; + padding: 6px 12px; + font-size: 12px; + font-family: 'Courier New', monospace; + cursor: pointer; + transition: all 0.2s; + border-radius: 4px; + } + button:hover { border-color: #606060; background: #252525; } + button:disabled { opacity: 0.3; cursor: not-allowed; } + video { display: none; } .drop-zone:hover { border-color: #606060; background: #252525; } .drop-zone.active { border-color: #4a9eff; background: #1a2a3a; } .drop-zone.error { border-color: #ff4a4a; background: #3a1a1a; } @@ -97,7 +121,7 @@ background: #1a1a1a; } .main.drop-active::after { - content: 'Drop PNG image here'; + content: 'Drop PNG/video here'; position: absolute; inset: 24px; display: flex; @@ -267,6 +291,11 @@

CNN v2 Testing Tool

+
+ + + +
@@ -282,6 +311,7 @@
+
@@ -316,7 +346,7 @@
- Drop PNG image anywhere to begin + Drop PNG/video anywhere to begin [SPACE] Original | [D] Diff (×10)
@@ -572,12 +602,16 @@ class CNNTester { this.status = document.getElementById('status'); this.console = document.getElementById('console'); this.image = null; + this.video = document.getElementById('videoSource'); this.weights = null; this.viewMode = 0; this.blendAmount = 1.0; this.depth = 1.0; this.currentLayerIdx = null; this.currentChannelOffset = null; + this.isVideo = false; + this.fps = 30; + this.isProcessing = false; this.init(); } @@ -690,8 +724,12 @@ class CNNTester { async loadImage(file) { const img = await createImageBitmap(file); this.image = img; + this.isVideo = false; this.canvas.width = img.width; this.canvas.height = img.height; + document.getElementById('playPauseBtn').disabled = true; + document.getElementById('stepBackBtn').disabled = true; + document.getElementById('stepForwardBtn').disabled = true; this.log(`Loaded image: ${file.name} (${img.width}×${img.height})`); if (this.weights) { this.setStatus(`Ready: ${img.width}×${img.height}`); @@ -702,6 +740,86 @@ class CNNTester { } } + // Video loading: wait for metadata, then first frame decode (readyState≥2) + async loadVideo(file) { + return new Promise((resolve, reject) => { + this.video.src = URL.createObjectURL(file); + + this.video.onloadedmetadata = () => { + const w = this.video.videoWidth; + const h = this.video.videoHeight; + if (w === 0 || h === 0) { + reject(new Error('Video has invalid dimensions')); + return; + } + + this.isVideo = true; + this.canvas.width = w; + this.canvas.height = h; + this.fps = 30; + this.log(`Loaded video: ${file.name} (${w}×${h}, ${this.video.duration.toFixed(1)}s)`); + + // Enable video controls + ['playPauseBtn', 'stepBackBtn', 'stepForwardBtn'].forEach(id => + document.getElementById(id).disabled = false + ); + + // Set up event handlers + this.video.onpause = () => { document.getElementById('playPauseBtn').textContent = 'Play'; }; + this.video.onplay = () => { document.getElementById('playPauseBtn').textContent = 'Pause'; this.playbackLoop(); }; + + // Wait for first frame to be decoded before displaying + const displayFirstFrame = () => { + this.video.onseeked = () => { if (!this.isProcessing) this.processVideoFrame(); }; + if (this.video.readyState >= 2) { // HAVE_CURRENT_DATA or better + if (this.weights) { + this.setStatus(`Ready: ${w}×${h}`); + this.processVideoFrame().then(() => resolve()); + } else { + this.setStatus(`Video loaded - drop .bin weights to process`); + this.displayOriginal(); + resolve(); + } + } else { + setTimeout(displayFirstFrame, 50); // Poll until frame ready + } + }; + + this.video.onseeked = displayFirstFrame; + this.video.currentTime = 0; + }; + + this.video.onerror = () => reject(new Error('Failed to load video')); + }); + } + + // Video playback loop (non-realtime, drops frames if CNN slow) + playbackLoop() { + if (this.video.paused || this.video.ended) return; + if (!this.isProcessing) this.processVideoFrame(); + requestAnimationFrame(() => this.playbackLoop()); + } + + // Process current video frame through CNN pipeline + async processVideoFrame() { + if (!this.weights || this.isProcessing) return; + this.isProcessing = true; + await this.run(); + this.isProcessing = false; + } + + // Video controls + togglePlayPause() { + this.video.paused ? this.video.play() : this.video.pause(); + } + + stepFrame(direction) { + if (!this.isVideo) return; + this.video.pause(); + this.video.currentTime = Math.max(0, Math.min(this.video.duration, + this.video.currentTime + direction / this.fps)); + } + async loadWeights(file) { const buffer = await file.arrayBuffer(); this.weights = this.parseWeights(buffer); @@ -776,9 +894,11 @@ class CNNTester { } displayOriginal() { - if (!this.image || !this.device) return; + const source = this.isVideo ? this.video : this.image; + if (!source || !this.device) return; - const { width, height } = this.image; + const width = this.isVideo ? this.video.videoWidth : this.image.width; + const height = this.isVideo ? this.video.videoHeight : this.image.height; this.context.configure({ device: this.device, format: this.format }); const inputTex = this.device.createTexture({ @@ -788,7 +908,7 @@ class CNNTester { }); this.device.queue.copyExternalImageToTexture( - { source: this.image }, + { source: source }, { texture: inputTex }, [width, height] ); @@ -843,9 +963,14 @@ class CNNTester { this.device.queue.submit([encoder.finish()]); } + // Run CNN inference pipeline on current source (image or video frame) async run() { const t0 = performance.now(); - const { width, height } = this.image; + const source = this.isVideo ? this.video : this.image; + if (!source) return; + const width = this.isVideo ? this.video.videoWidth : this.image.width; + const height = this.isVideo ? this.video.videoHeight : this.image.height; + this.log(`Running CNN pipeline (${this.weights.layers.length} layers)...`); this.context.configure({ device: this.device, format: this.format }); @@ -859,7 +984,7 @@ class CNNTester { }); this.device.queue.copyExternalImageToTexture( - { source: this.image }, + { source: source }, { texture: this.inputTexture }, [width, height] ); @@ -1067,6 +1192,9 @@ class CNNTester { } updateLayerVizPanel() { + // Optimization: Skip layer viz updates during video playback (~5-10ms saved per frame) + if (this.isVideo && !this.video.paused) return; + const panel = document.getElementById('layerViz'); if (!this.layerOutputs || this.layerOutputs.length === 0) { @@ -1153,7 +1281,8 @@ class CNNTester { const layerName = layerIdx === 0 ? `Static Features (${channelOffset}-${channelOffset + 3})` : `Layer ${layerIdx - 1}`; const layerTex = this.layerOutputs[layerIdx]; - const { width, height } = this.image; + const width = this.isVideo ? this.video.videoWidth : this.image.width; + const height = this.isVideo ? this.video.videoHeight : this.image.height; this.log(`Visualizing ${layerName} activations (${width}×${height})`); @@ -1265,7 +1394,8 @@ class CNNTester { const zoomCanvas = document.getElementById('zoomCanvas'); if (!zoomCanvas) return; - const { width, height } = this.image; + const width = this.isVideo ? this.video.videoWidth : this.image.width; + const height = this.isVideo ? this.video.videoHeight : this.image.height; const zoomSize = 32; // Show 32x32 area zoomCanvas.width = zoomSize; zoomCanvas.height = zoomSize; @@ -1301,7 +1431,8 @@ class CNNTester { } const halfSize = Math.floor(zoomSize / 2); - const { width, height } = this.image; + const width = this.isVideo ? this.video.videoWidth : this.image.width; + const height = this.isVideo ? this.video.videoHeight : this.image.height; // Create shader for zoomed view (samples 4 channels and displays as 2x2 grid) const zoomShader = ` @@ -1520,7 +1651,7 @@ function setupDropZone(id, callback) { }); } -// Whole window drop for PNG images +// Whole window drop for PNG images and videos const mainArea = document.getElementById('mainDrop'); ['dragenter', 'dragover', 'dragleave', 'drop'].forEach(e => { mainArea.addEventListener(e, ev => { ev.preventDefault(); ev.stopPropagation(); }); @@ -1529,11 +1660,18 @@ const mainArea = document.getElementById('mainDrop'); ['dragleave', 'drop'].forEach(e => mainArea.addEventListener(e, () => mainArea.classList.remove('drop-active'))); mainArea.addEventListener('drop', e => { const file = e.dataTransfer.files[0]; - if (file && file.type.startsWith('image/')) { - tester.loadImage(file).catch(err => { - tester.setStatus(err.message, true); - tester.log(err.message, 'error'); - }); + if (file) { + if (file.type.startsWith('image/')) { + tester.loadImage(file).catch(err => { + tester.setStatus(err.message, true); + tester.log(err.message, 'error'); + }); + } else if (file.type.startsWith('video/')) { + tester.loadVideo(file).catch(err => { + tester.setStatus(err.message, true); + tester.log(err.message, 'error'); + }); + } } }); @@ -1554,7 +1692,7 @@ document.getElementById('weightsFile').addEventListener('change', e => { document.getElementById('blend').addEventListener('input', e => { tester.blendAmount = parseFloat(e.target.value); document.getElementById('blendValue').textContent = e.target.value; - if (tester.image && tester.weights) { + if ((tester.image || tester.isVideo) && tester.weights) { tester.log(`Blend changed to ${e.target.value}`); tester.run(); } @@ -1562,9 +1700,13 @@ document.getElementById('blend').addEventListener('input', e => { document.getElementById('depth').addEventListener('input', e => { tester.depth = parseFloat(e.target.value); - if (tester.image && tester.weights) tester.run(); + if ((tester.image || tester.isVideo) && tester.weights) tester.run(); }); +document.getElementById('playPauseBtn').addEventListener('click', () => tester.togglePlayPause()); +document.getElementById('stepBackBtn').addEventListener('click', () => tester.stepFrame(-1)); +document.getElementById('stepForwardBtn').addEventListener('click', () => tester.stepFrame(1)); + document.addEventListener('keydown', e => { if (e.code === 'Space') { e.preventDefault(); @@ -1575,10 +1717,11 @@ document.addEventListener('keydown', e => { } const modeName = ['CNN Output', 'Original', 'Diff (×10)'][tester.viewMode]; document.getElementById('viewMode').textContent = modeName; - if (tester.image && tester.weights) { + if ((tester.image || tester.isVideo) && tester.weights) { tester.log(`View mode: ${modeName}`); tester.updateDisplay(); - const { width, height } = tester.image; + const width = tester.isVideo ? tester.video.videoWidth : tester.image.width; + const height = tester.isVideo ? tester.video.videoHeight : tester.image.height; tester.setStatus(`${width}×${height} | ${modeName}`); } } else if (e.code === 'KeyD') { @@ -1590,10 +1733,11 @@ document.addEventListener('keydown', e => { } const modeName = ['CNN Output', 'Original', 'Diff (×10)'][tester.viewMode]; document.getElementById('viewMode').textContent = modeName; - if (tester.image && tester.weights) { + if ((tester.image || tester.isVideo) && tester.weights) { tester.log(`View mode: ${modeName}`); tester.updateDisplay(); - const { width, height } = tester.image; + const width = tester.isVideo ? tester.video.videoWidth : tester.image.width; + const height = tester.isVideo ? tester.video.videoHeight : tester.image.height; tester.setStatus(`${width}×${height} | ${modeName}`); } } -- cgit v1.2.3