diff options
| author | skal <pascal.massimino@gmail.com> | 2026-02-17 16:12:21 +0100 |
|---|---|---|
| committer | skal <pascal.massimino@gmail.com> | 2026-02-17 16:12:21 +0100 |
| commit | 03579c4a33ab3955ff9924a6dcd882fe91dd9aaa (patch) | |
| tree | be458d2ac4bc0d7160be8a18526b4e9157af33a5 /tools/mq_editor/viewer.js | |
| parent | e3f0b002c0998c8553e782273b254869107ffc0f (diff) | |
feat(mq_editor): Phase 1 - MQ extraction and visualization (SPECTRAL_BRUSH_2)
Implement McAulay-Quatieri sinusoidal analysis tool for audio compression.
New files:
- doc/SPECTRAL_BRUSH_2.md: Complete design doc (MQ algorithm, data format, synthesis, roadmap)
- tools/mq_editor/index.html: Web UI (file loader, params, canvas)
- tools/mq_editor/fft.js: Radix-2 Cooley-Tukey FFT (from spectral_editor)
- tools/mq_editor/mq_extract.js: MQ algorithm (peak detection, tracking, bezier fitting)
- tools/mq_editor/viewer.js: Visualization (spectrogram, partials, zoom, axes)
- tools/mq_editor/README.md: Usage and implementation status
Features:
- Load WAV → extract sinusoidal partials → fit cubic bezier curves
- Time-frequency spectrogram with hot colormap (0-16 kHz)
- Horizontal zoom (mousewheel) around mouse position
- Axis ticks with labels (time: seconds, freq: Hz/kHz)
- Mouse tooltip showing time/frequency coordinates
- Real-time adjustable MQ parameters (FFT size, hop, threshold)
Algorithm:
- STFT with Hann windows (2048 FFT, 512 hop)
- Peak detection with parabolic interpolation
- Birth/death/continuation tracking (50 Hz tolerance)
- Cubic bezier fitting (4 control points per trajectory)
Next: Phase 2 (JS synthesizer for audio preview)
handoff(Claude): MQ editor Phase 1 complete. Ready for synthesis implementation.
Diffstat (limited to 'tools/mq_editor/viewer.js')
| -rw-r--r-- | tools/mq_editor/viewer.js | 376 |
1 files changed, 376 insertions, 0 deletions
diff --git a/tools/mq_editor/viewer.js b/tools/mq_editor/viewer.js new file mode 100644 index 0000000..1b2f5bf --- /dev/null +++ b/tools/mq_editor/viewer.js @@ -0,0 +1,376 @@ +// Spectrogram Viewer +// Handles all visualization: spectrogram, partials, zoom, mouse interaction + +class SpectrogramViewer { + constructor(canvas, audioBuffer) { + this.canvas = canvas; + this.ctx = canvas.getContext('2d'); + this.audioBuffer = audioBuffer; + this.partials = []; + + // View state (time only, frequency fixed) + this.timeStart = 0; + this.timeEnd = audioBuffer.duration; + this.freqStart = 0; + this.freqEnd = 16000; // Fixed + + // Tooltip + this.tooltip = document.getElementById('tooltip'); + + // Setup event handlers + this.setupMouseHandlers(); + + // Initial render + this.render(); + } + + setPartials(partials) { + this.partials = partials; + this.render(); + } + + reset() { + this.timeStart = 0; + this.timeEnd = this.audioBuffer.duration; + this.render(); + } + + render() { + this.renderSpectrogram(); + this.renderPartials(); + this.drawAxes(); + } + + // Render spectrogram background + renderSpectrogram() { + const {canvas, ctx, audioBuffer} = this; + const width = canvas.width; + const height = canvas.height; + + ctx.fillStyle = '#000'; + ctx.fillRect(0, 0, width, height); + + const signal = getMono(audioBuffer); + const fftSize = 2048; + const hopSize = 512; + const sampleRate = audioBuffer.sampleRate; + + const numFrames = Math.floor((signal.length - fftSize) / hopSize); + + // Compute one FFT per ~4 pixels for wider bars + const pixelsPerFrame = 4; + const numDisplayFrames = Math.floor(width / pixelsPerFrame); + + // Map view bounds to frame indices + const startFrameIdx = Math.floor(this.timeStart * sampleRate / hopSize); + const endFrameIdx = Math.floor(this.timeEnd * sampleRate / hopSize); + const visibleFrames = endFrameIdx - startFrameIdx; + const frameStep = Math.max(1, Math.floor(visibleFrames / numDisplayFrames)); + + for (let displayIdx = 0; displayIdx < numDisplayFrames; ++displayIdx) { + const frameIdx = startFrameIdx + displayIdx * frameStep; + if (frameIdx >= numFrames) break; + + const offset = frameIdx * hopSize; + if (offset + fftSize > signal.length) break; + + const frame = signal.slice(offset, offset + fftSize); + + // Windowing + const windowed = new Float32Array(fftSize); + for (let i = 0; i < fftSize; ++i) { + const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / fftSize); + windowed[i] = frame[i] * w; + } + + // FFT + const spectrum = realFFT(windowed); + + // Draw as vertical bar + const xStart = displayIdx * pixelsPerFrame; + const xEnd = Math.min(xStart + pixelsPerFrame, width); + + // Draw frequency bins + const numBins = fftSize / 2; + for (let bin = 0; bin < numBins; ++bin) { + const freq = bin * sampleRate / fftSize; + if (freq < this.freqStart || freq > this.freqEnd) continue; + + const re = spectrum[bin * 2]; + const im = spectrum[bin * 2 + 1]; + const mag = Math.sqrt(re * re + im * im); + const magDB = 20 * Math.log10(Math.max(mag, 1e-10)); + + const normalized = (magDB + 80) / 60; + const intensity = Math.max(0, Math.min(1, normalized)); + + const freqNorm = (freq - this.freqStart) / (this.freqEnd - this.freqStart); + const y = Math.floor(height - freqNorm * height); + if (y < 0 || y >= height) continue; + + const color = this.getSpectrogramColor(intensity); + ctx.fillStyle = `rgb(${color.r},${color.g},${color.b})`; + ctx.fillRect(xStart, y, xEnd - xStart, 1); + } + } + } + + // Render extracted partials + renderPartials() { + const {ctx, canvas, partials} = this; + const width = canvas.width; + const height = canvas.height; + + const colors = [ + '#f44', '#4f4', '#44f', '#ff4', '#f4f', '#4ff', + '#fa4', '#4fa', '#a4f', '#af4', '#f4a', '#4af' + ]; + + const timeDuration = this.timeEnd - this.timeStart; + const freqRange = this.freqEnd - this.freqStart; + + for (let p = 0; p < partials.length; ++p) { + const partial = partials[p]; + const color = colors[p % colors.length]; + + // Draw raw trajectory + ctx.strokeStyle = color + '44'; + ctx.lineWidth = 1; + ctx.beginPath(); + + let started = false; + for (let i = 0; i < partial.times.length; ++i) { + const t = partial.times[i]; + const f = partial.freqs[i]; + + if (t < this.timeStart || t > this.timeEnd) continue; + if (f < this.freqStart || f > this.freqEnd) continue; + + const x = (t - this.timeStart) / timeDuration * width; + const y = height - (f - this.freqStart) / freqRange * height; + + if (!started) { + ctx.moveTo(x, y); + started = true; + } else { + ctx.lineTo(x, y); + } + } + + if (started) ctx.stroke(); + + // Draw bezier curve + if (partial.freqCurve) { + ctx.strokeStyle = color; + ctx.lineWidth = 2; + ctx.beginPath(); + + const curve = partial.freqCurve; + const numSteps = 50; + + started = false; + for (let i = 0; i <= numSteps; ++i) { + const t = curve.t0 + (curve.t3 - curve.t0) * i / numSteps; + const freq = evalBezier(curve, t); + + if (t < this.timeStart || t > this.timeEnd) continue; + if (freq < this.freqStart || freq > this.freqEnd) continue; + + const x = (t - this.timeStart) / timeDuration * width; + const y = height - (freq - this.freqStart) / freqRange * height; + + if (!started) { + ctx.moveTo(x, y); + started = true; + } else { + ctx.lineTo(x, y); + } + } + + if (started) ctx.stroke(); + + // Draw control points + ctx.fillStyle = color; + this.drawControlPoint(curve.t0, curve.v0); + this.drawControlPoint(curve.t1, curve.v1); + this.drawControlPoint(curve.t2, curve.v2); + this.drawControlPoint(curve.t3, curve.v3); + } + } + } + + // Draw control point + drawControlPoint(t, v) { + if (t < this.timeStart || t > this.timeEnd) return; + if (v < this.freqStart || v > this.freqEnd) return; + + const timeDuration = this.timeEnd - this.timeStart; + const freqRange = this.freqEnd - this.freqStart; + + const x = (t - this.timeStart) / timeDuration * this.canvas.width; + const y = this.canvas.height - (v - this.freqStart) / freqRange * this.canvas.height; + + this.ctx.beginPath(); + this.ctx.arc(x, y, 4, 0, 2 * Math.PI); + this.ctx.fill(); + + this.ctx.strokeStyle = '#fff'; + this.ctx.lineWidth = 1; + this.ctx.stroke(); + } + + // Draw axes with ticks and labels + drawAxes() { + const {ctx, canvas} = this; + const width = canvas.width; + const height = canvas.height; + + ctx.strokeStyle = '#666'; + ctx.fillStyle = '#aaa'; + ctx.font = '11px monospace'; + ctx.lineWidth = 1; + + const timeDuration = this.timeEnd - this.timeStart; + const freqRange = this.freqEnd - this.freqStart; + + // Time axis + const timeStep = this.getAxisStep(timeDuration); + let t = Math.ceil(this.timeStart / timeStep) * timeStep; + while (t <= this.timeEnd) { + const x = (t - this.timeStart) / timeDuration * width; + + ctx.beginPath(); + ctx.moveTo(x, 0); + ctx.lineTo(x, height); + ctx.stroke(); + + ctx.fillText(t.toFixed(2) + 's', x + 2, height - 4); + t += timeStep; + } + + // Frequency axis + const freqStep = this.getAxisStep(freqRange); + let f = Math.ceil(this.freqStart / freqStep) * freqStep; + while (f <= this.freqEnd) { + const y = height - (f - this.freqStart) / freqRange * height; + + ctx.beginPath(); + ctx.moveTo(0, y); + ctx.lineTo(width, y); + ctx.stroke(); + + const label = f >= 1000 ? (f/1000).toFixed(1) + 'k' : f.toFixed(0); + ctx.fillText(label + 'Hz', 2, y - 2); + f += freqStep; + } + } + + // Setup mouse event handlers + setupMouseHandlers() { + const {canvas, tooltip} = this; + + // Mouse move (tooltip) + canvas.addEventListener('mousemove', (e) => { + const rect = canvas.getBoundingClientRect(); + const x = e.clientX - rect.left; + const y = e.clientY - rect.top; + + const time = this.canvasToTime(x); + const freq = this.canvasToFreq(y); + + tooltip.style.left = (e.clientX + 10) + 'px'; + tooltip.style.top = (e.clientY + 10) + 'px'; + tooltip.style.display = 'block'; + tooltip.textContent = `${time.toFixed(3)}s, ${freq.toFixed(1)}Hz`; + }); + + canvas.addEventListener('mouseleave', () => { + tooltip.style.display = 'none'; + }); + + // Mouse wheel (horizontal zoom only) + canvas.addEventListener('wheel', (e) => { + e.preventDefault(); + + const rect = canvas.getBoundingClientRect(); + const x = e.clientX - rect.left; + + // Get mouse position in time space + const mouseTime = this.canvasToTime(x); + + // Zoom factor + const zoomFactor = e.deltaY > 0 ? 1.2 : 0.8; + + // Zoom time around mouse position + const timeDuration = this.timeEnd - this.timeStart; + const newTimeDuration = timeDuration * zoomFactor; + const timeRatio = (mouseTime - this.timeStart) / timeDuration; + + this.timeStart = mouseTime - newTimeDuration * timeRatio; + this.timeEnd = mouseTime + newTimeDuration * (1 - timeRatio); + + // Clamp time bounds + if (this.timeStart < 0) { + this.timeEnd -= this.timeStart; + this.timeStart = 0; + } + if (this.timeEnd > this.audioBuffer.duration) { + this.timeStart -= (this.timeEnd - this.audioBuffer.duration); + this.timeEnd = this.audioBuffer.duration; + } + + // Re-render + this.render(); + }); + } + + // Coordinate conversion + canvasToTime(x) { + return this.timeStart + (x / this.canvas.width) * (this.timeEnd - this.timeStart); + } + + canvasToFreq(y) { + return this.freqEnd - (y / this.canvas.height) * (this.freqEnd - this.freqStart); + } + + // Utilities + getAxisStep(range) { + const steps = [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000]; + const targetSteps = 8; + const targetStep = range / targetSteps; + + for (const step of steps) { + if (step >= targetStep) return step; + } + + return steps[steps.length - 1]; + } + + getSpectrogramColor(intensity) { + if (intensity < 0.25) { + const t = intensity / 0.25; + return {r: 0, g: 0, b: Math.floor(t * 128)}; + } else if (intensity < 0.5) { + const t = (intensity - 0.25) / 0.25; + return {r: 0, g: Math.floor(t * 128), b: 128}; + } else if (intensity < 0.75) { + const t = (intensity - 0.5) / 0.25; + return {r: Math.floor(t * 255), g: 128 + Math.floor(t * 127), b: 128 - Math.floor(t * 128)}; + } else { + const t = (intensity - 0.75) / 0.25; + return {r: 255, g: 255 - Math.floor(t * 128), b: 0}; + } + } +} + +// Bezier evaluation (shared utility) +function evalBezier(curve, t) { + let u = (t - curve.t0) / (curve.t3 - curve.t0); + u = Math.max(0, Math.min(1, u)); + + const u1 = 1 - u; + return u1*u1*u1 * curve.v0 + + 3*u1*u1*u * curve.v1 + + 3*u1*u*u * curve.v2 + + u*u*u * curve.v3; +} |
