diff options
| author | skal <pascal.massimino@gmail.com> | 2026-02-18 17:59:49 +0100 |
|---|---|---|
| committer | skal <pascal.massimino@gmail.com> | 2026-02-18 17:59:49 +0100 |
| commit | 49e4e374bc51517d8119e4c82d46e3a94ea0b75b (patch) | |
| tree | e0ac136257faba9dc322978a5cd1df52012541c5 /tools/mq_editor | |
| parent | b9f6429b6cc741c5240c1fea8bbf4c6244e4e5d1 (diff) | |
feat(mq_editor): add explore mode for interactive partial tracking
Hover to preview a tracked partial from mouse position (peak-snapped,
forward+backward MQ tracking). Click to commit. Toggle with ⊕ Explore
button or X key. Escape exits explore mode.
handoff(Gemini): explore mode added in mq_extract.trackFromSeed + viewer.js/app.js
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Diffstat (limited to 'tools/mq_editor')
| -rw-r--r-- | tools/mq_editor/app.js | 34 | ||||
| -rw-r--r-- | tools/mq_editor/index.html | 2 | ||||
| -rw-r--r-- | tools/mq_editor/mq_extract.js | 96 | ||||
| -rw-r--r-- | tools/mq_editor/viewer.js | 74 |
4 files changed, 199 insertions, 7 deletions
diff --git a/tools/mq_editor/app.js b/tools/mq_editor/app.js index b0b578b..41df49a 100644 --- a/tools/mq_editor/app.js +++ b/tools/mq_editor/app.js @@ -45,6 +45,14 @@ let audioContext = null; let currentSource = null; let extractedPartials = null; let stftCache = null; +let exploreModeActive = false; + +function setExploreMode(enabled) { + exploreModeActive = enabled; + const btn = document.getElementById('exploreBtn'); + btn.classList.toggle('explore-active', enabled); + if (viewer) viewer.setExploreMode(enabled); +} // Undo/redo const undoStack = []; @@ -152,6 +160,26 @@ function loadAudioBuffer(buffer, label) { viewer.onPartialSelect = (i) => editor.onPartialSelect(i); viewer.onRender = () => editor.onRender(); viewer.onBeforeChange = pushUndo; + viewer.onExploreMove = (time, freq) => { + if (!viewer.frames || viewer.frames.length === 0) return; + const params = { + hopSize: Math.max(64, parseInt(hopSize.value) || 64), + sampleRate: audioBuffer.sampleRate, + deathAge: parseInt(deathAgeEl.value), + phaseErrorWeight: parseFloat(phaseErrorWeightEl.value), + }; + viewer.setPreviewPartial(trackFromSeed(viewer.frames, time, freq, params)); + }; + viewer.onExploreCommit = (partial) => { + if (!extractedPartials) extractedPartials = []; + pushUndo(); + extractedPartials.unshift(partial); + editor.setPartials(extractedPartials); + viewer.setPartials(extractedPartials); + viewer.setKeepCount(getKeepCount()); + viewer.selectPartial(0); + setStatus(`Explore: added partial (${extractedPartials.length} total)`, 'info'); + }; if (label.startsWith('Test WAV')) validateTestWAVPeaks(stftCache); }, 10); } @@ -256,6 +284,7 @@ function runExtraction() { autoSpreadAllBtn.disabled = false; document.getElementById('newPartialBtn').disabled = false; document.getElementById('clearAllBtn').disabled = false; + document.getElementById('exploreBtn').disabled = false; undoStack.length = 0; redoStack.length = 0; _updateUndoRedoBtns(); }, 50); } @@ -305,6 +334,7 @@ function clearAllPartials() { document.getElementById('newPartialBtn').addEventListener('click', createNewPartial); document.getElementById('clearAllBtn').addEventListener('click', clearAllPartials); +document.getElementById('exploreBtn').addEventListener('click', () => setExploreMode(!exploreModeActive)); document.getElementById('undoBtn').addEventListener('click', undo); document.getElementById('redoBtn').addEventListener('click', redo); @@ -480,7 +510,11 @@ document.addEventListener('keydown', (e) => { } else if (e.code === 'KeyE') { e.preventDefault(); if (!extractBtn.disabled) extractBtn.click(); + } else if (e.code === 'KeyX' && !e.ctrlKey && !e.metaKey) { + e.preventDefault(); + if (!document.getElementById('exploreBtn').disabled) setExploreMode(!exploreModeActive); } else if (e.code === 'Escape') { + if (exploreModeActive) { setExploreMode(false); return; } if (viewer) viewer.selectPartial(-1); } }); diff --git a/tools/mq_editor/index.html b/tools/mq_editor/index.html index 6cf691f..9d50044 100644 --- a/tools/mq_editor/index.html +++ b/tools/mq_editor/index.html @@ -41,6 +41,7 @@ button:hover { background: #4a4a4a; } button:disabled { opacity: 0.5; cursor: not-allowed; } #extractBtn { background: #666; color: #fff; font-weight: bold; border-color: #888; } + button.explore-active { background: #554; border-color: #aa8; color: #ffd; } input[type="file"] { display: none; } .params { display: flex; @@ -286,6 +287,7 @@ <button id="stopBtn" disabled>■ Stop</button> <button id="newPartialBtn" disabled>+ Partial</button> <button id="clearAllBtn" disabled>✕ Clear All</button> + <button id="exploreBtn" disabled>⊕ Explore</button> <button id="undoBtn" disabled>↩ Undo</button> <button id="redoBtn" disabled>↪ Redo</button> diff --git a/tools/mq_editor/mq_extract.js b/tools/mq_editor/mq_extract.js index 97191e2..8f6961f 100644 --- a/tools/mq_editor/mq_extract.js +++ b/tools/mq_editor/mq_extract.js @@ -336,6 +336,102 @@ function autodetectSpread(partial, stftCache, fftSize, sampleRate) { return {spread_above, spread_below}; } +// Track a single partial starting from a (time, freq) seed position. +// Snaps to nearest spectral peak, then tracks forward and backward. +// Returns a partial object (with freqCurve), or null if no peak found near seed. +function trackFromSeed(frames, seedTime, seedFreq, params) { + if (!frames || frames.length === 0) return null; + + // Find nearest frame to seedTime + let seedFrameIdx = 0; + let bestDt = Infinity; + for (let i = 0; i < frames.length; ++i) { + const dt = Math.abs(frames[i].time - seedTime); + if (dt < bestDt) { bestDt = dt; seedFrameIdx = i; } + } + + // Snap to nearest spectral peak within 10% freq tolerance + const seedFrame = frames[seedFrameIdx]; + const snapTol = Math.max(seedFreq * 0.10, 50); + let seedPeak = null, bestDist = snapTol; + for (const pk of seedFrame.peaks) { + const d = Math.abs(pk.freq - seedFreq); + if (d < bestDist) { bestDist = d; seedPeak = pk; } + } + if (!seedPeak) return null; + + const { hopSize, sampleRate, deathAge = 5, phaseErrorWeight = 2.0 } = params; + const trackingRatio = 0.05; + const minTrackingHz = 20; + + // Forward pass from seed frame + const times = [seedFrame.time]; + const freqs = [seedPeak.freq]; + const amps = [seedPeak.amp]; + const phases = [seedPeak.phase]; + + let fwdFreq = seedPeak.freq, fwdPhase = seedPeak.phase, fwdVel = 0, fwdAge = 0; + for (let i = seedFrameIdx + 1; i < frames.length; ++i) { + const predicted = fwdFreq + fwdVel; + const predPhase = fwdPhase + 2 * Math.PI * fwdFreq * (fwdAge + 1) * hopSize / sampleRate; + const tol = Math.max(predicted * trackingRatio, minTrackingHz); + const { bestIdx } = findBestPeak(frames[i].peaks, new Set(), predicted, predPhase, tol, phaseErrorWeight); + if (bestIdx >= 0) { + const pk = frames[i].peaks[bestIdx]; + times.push(frames[i].time); + freqs.push(pk.freq); + amps.push(pk.amp); + phases.push(pk.phase); + fwdVel = pk.freq - fwdFreq; + fwdFreq = pk.freq; fwdPhase = pk.phase; fwdAge = 0; + } else { + fwdAge++; + if (fwdAge > deathAge) break; + } + } + + // Backward pass from seed frame + const bwdTimes = [], bwdFreqs = [], bwdAmps = [], bwdPhases = []; + let bwdFreq = seedPeak.freq, bwdAge = 0; + for (let i = seedFrameIdx - 1; i >= 0; --i) { + const tol = Math.max(bwdFreq * trackingRatio, minTrackingHz); + let bestIdx = -1, bDist = tol; + for (let j = 0; j < frames[i].peaks.length; ++j) { + const d = Math.abs(frames[i].peaks[j].freq - bwdFreq); + if (d < bDist) { bDist = d; bestIdx = j; } + } + if (bestIdx >= 0) { + const pk = frames[i].peaks[bestIdx]; + bwdTimes.unshift(frames[i].time); + bwdFreqs.unshift(pk.freq); + bwdAmps.unshift(pk.amp); + bwdPhases.unshift(pk.phase); + bwdFreq = pk.freq; bwdAge = 0; + } else { + bwdAge++; + if (bwdAge > deathAge) break; + } + } + + const allTimes = [...bwdTimes, ...times]; + const allFreqs = [...bwdFreqs, ...freqs]; + const allAmps = [...bwdAmps, ...amps]; + const allPhases = [...bwdPhases, ...phases]; + + if (allTimes.length < 2) return null; + + const freqCurve = fitBezier(allTimes, allFreqs); + const ac = fitBezier(allTimes, allAmps); + freqCurve.a0 = ac.v0; freqCurve.a1 = ac.v1; + freqCurve.a2 = ac.v2; freqCurve.a3 = ac.v3; + + return { + times: allTimes, freqs: allFreqs, amps: allAmps, phases: allPhases, + muted: false, freqCurve, + replicas: { decay_alpha: 0.1, jitter: 0.05, spread_above: 0.02, spread_below: 0.02 }, + }; +} + // Fit cubic bezier to trajectory using least-squares for inner control points function fitBezier(times, values) { const n = times.length - 1; diff --git a/tools/mq_editor/viewer.js b/tools/mq_editor/viewer.js index fea5cd0..8f97c3b 100644 --- a/tools/mq_editor/viewer.js +++ b/tools/mq_editor/viewer.js @@ -57,6 +57,12 @@ class SpectrogramViewer { this.onRender = null; // callback() called after each render (for synced panels) this.onBeforeChange = null; // callback() called before any mutation (for undo/redo) + // Explore mode + this.exploreMode = false; + this.previewPartial = null; + this.onExploreMove = null; // callback(time, freq) + this.onExploreCommit = null; // callback(partial) + // Setup event handlers this.setupMouseHandlers(); @@ -378,12 +384,51 @@ class SpectrogramViewer { const h = this.cursorCanvas.height; ctx.clearRect(0, 0, this.cursorCanvas.width, h); if (x < 0) return; - ctx.strokeStyle = 'rgba(255, 60, 60, 0.7)'; + ctx.strokeStyle = this.exploreMode ? 'rgba(255,160,0,0.8)' : 'rgba(255,60,60,0.7)'; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, h); ctx.stroke(); + if (this.exploreMode && this.previewPartial) { + this._drawPreviewPartial(ctx, this.previewPartial); + } + } + + setExploreMode(enabled) { + this.exploreMode = enabled; + if (!enabled) this.previewPartial = null; + this.drawMouseCursor(this.mouseX); + this.canvas.style.cursor = enabled ? 'cell' : 'crosshair'; + } + + setPreviewPartial(partial) { + this.previewPartial = partial; + this.drawMouseCursor(this.mouseX); + } + + _drawPreviewPartial(ctx, partial) { + const curve = partial.freqCurve; + if (!curve) return; + ctx.save(); + ctx.strokeStyle = 'rgba(255,160,0,0.9)'; + ctx.lineWidth = 2; + ctx.setLineDash([6, 3]); + ctx.shadowColor = 'rgba(255,160,0,0.5)'; + ctx.shadowBlur = 6; + ctx.beginPath(); + let started = false; + for (let i = 0; i <= 80; ++i) { + const t = curve.t0 + (curve.t3 - curve.t0) * i / 80; + const freq = evalBezier(curve, t); + if (t < this.t_view_min || t > this.t_view_max) continue; + if (freq < this.freqStart || freq > this.freqEnd) continue; + const px = this.timeToX(t); + const py = this.freqToY(freq); + if (!started) { ctx.moveTo(px, py); started = true; } else ctx.lineTo(px, py); + } + if (started) ctx.stroke(); + ctx.restore(); } drawPlayhead() { @@ -566,6 +611,14 @@ class SpectrogramViewer { canvas.addEventListener('mousedown', (e) => { const {x, y} = getCanvasCoords(e, canvas); + // Explore mode: commit preview on click + if (this.exploreMode) { + if (this.previewPartial && this.onExploreCommit) { + this.onExploreCommit(this.previewPartial); + } + return; + } + // Check control point drag on selected partial if (this.selectedPartial >= 0 && this.selectedPartial < this.partials.length) { const ptIdx = this.hitTestControlPoint(x, y, this.partials[this.selectedPartial]); @@ -614,6 +667,11 @@ class SpectrogramViewer { const time = this.canvasToTime(x); const freq = this.canvasToFreq(y); + + if (this.exploreMode && this.onExploreMove) { + this.onExploreMove(time, freq); // may call setPreviewPartial → redraws cursor canvas + } + const intensity = this.getIntensityAt(time, freq); if (this.playheadTime < 0) { @@ -621,12 +679,14 @@ class SpectrogramViewer { this.renderSpectrum(); } - // Cursor hint for control points - if (this.selectedPartial >= 0 && this.selectedPartial < this.partials.length) { - const ptIdx = this.hitTestControlPoint(x, y, this.partials[this.selectedPartial]); - canvas.style.cursor = ptIdx >= 0 ? 'grab' : 'crosshair'; - } else { - canvas.style.cursor = 'crosshair'; + // Cursor hint for control points (skip in explore mode) + if (!this.exploreMode) { + if (this.selectedPartial >= 0 && this.selectedPartial < this.partials.length) { + const ptIdx = this.hitTestControlPoint(x, y, this.partials[this.selectedPartial]); + canvas.style.cursor = ptIdx >= 0 ? 'grab' : 'crosshair'; + } else { + canvas.style.cursor = 'crosshair'; + } } tooltip.style.left = (e.clientX + 10) + 'px'; |
