summaryrefslogtreecommitdiff
path: root/tools/mq_editor/mq_extract.js
diff options
context:
space:
mode:
authorskal <pascal.massimino@gmail.com>2026-02-17 16:12:21 +0100
committerskal <pascal.massimino@gmail.com>2026-02-17 16:12:21 +0100
commit03579c4a33ab3955ff9924a6dcd882fe91dd9aaa (patch)
treebe458d2ac4bc0d7160be8a18526b4e9157af33a5 /tools/mq_editor/mq_extract.js
parente3f0b002c0998c8553e782273b254869107ffc0f (diff)
feat(mq_editor): Phase 1 - MQ extraction and visualization (SPECTRAL_BRUSH_2)
Implement McAulay-Quatieri sinusoidal analysis tool for audio compression. New files: - doc/SPECTRAL_BRUSH_2.md: Complete design doc (MQ algorithm, data format, synthesis, roadmap) - tools/mq_editor/index.html: Web UI (file loader, params, canvas) - tools/mq_editor/fft.js: Radix-2 Cooley-Tukey FFT (from spectral_editor) - tools/mq_editor/mq_extract.js: MQ algorithm (peak detection, tracking, bezier fitting) - tools/mq_editor/viewer.js: Visualization (spectrogram, partials, zoom, axes) - tools/mq_editor/README.md: Usage and implementation status Features: - Load WAV → extract sinusoidal partials → fit cubic bezier curves - Time-frequency spectrogram with hot colormap (0-16 kHz) - Horizontal zoom (mousewheel) around mouse position - Axis ticks with labels (time: seconds, freq: Hz/kHz) - Mouse tooltip showing time/frequency coordinates - Real-time adjustable MQ parameters (FFT size, hop, threshold) Algorithm: - STFT with Hann windows (2048 FFT, 512 hop) - Peak detection with parabolic interpolation - Birth/death/continuation tracking (50 Hz tolerance) - Cubic bezier fitting (4 control points per trajectory) Next: Phase 2 (JS synthesizer for audio preview) handoff(Claude): MQ editor Phase 1 complete. Ready for synthesis implementation.
Diffstat (limited to 'tools/mq_editor/mq_extract.js')
-rw-r--r--tools/mq_editor/mq_extract.js216
1 files changed, 216 insertions, 0 deletions
diff --git a/tools/mq_editor/mq_extract.js b/tools/mq_editor/mq_extract.js
new file mode 100644
index 0000000..62b275c
--- /dev/null
+++ b/tools/mq_editor/mq_extract.js
@@ -0,0 +1,216 @@
+// MQ Extraction Algorithm
+// McAulay-Quatieri sinusoidal analysis
+
+// Extract partials from audio buffer
+function extractPartials(audioBuffer, params) {
+ const {fftSize, hopSize, threshold, sampleRate} = params;
+
+ // Get mono channel (mix to mono if stereo)
+ const signal = getMono(audioBuffer);
+ const numFrames = Math.floor((signal.length - fftSize) / hopSize);
+
+ // Analyze frames
+ const frames = [];
+ for (let i = 0; i < numFrames; ++i) {
+ const offset = i * hopSize;
+ const frame = signal.slice(offset, offset + fftSize);
+ const peaks = detectPeaks(frame, fftSize, sampleRate, threshold);
+ const time = offset / sampleRate;
+ frames.push({time, peaks});
+ }
+
+ // Track trajectories
+ const partials = trackPartials(frames, sampleRate);
+
+ // Fit bezier curves
+ for (const partial of partials) {
+ partial.freqCurve = fitBezier(partial.times, partial.freqs);
+ partial.ampCurve = fitBezier(partial.times, partial.amps);
+ }
+
+ return partials;
+}
+
+// Get mono signal
+function getMono(audioBuffer) {
+ const data = audioBuffer.getChannelData(0);
+ if (audioBuffer.numberOfChannels === 1) {
+ return data;
+ }
+
+ // Mix to mono
+ const left = audioBuffer.getChannelData(0);
+ const right = audioBuffer.getChannelData(1);
+ const mono = new Float32Array(left.length);
+ for (let i = 0; i < left.length; ++i) {
+ mono[i] = (left[i] + right[i]) * 0.5;
+ }
+ return mono;
+}
+
+// Detect peaks in FFT frame
+function detectPeaks(frame, fftSize, sampleRate, thresholdDB) {
+ // Apply Hann window
+ const windowed = new Float32Array(fftSize);
+ for (let i = 0; i < fftSize; ++i) {
+ const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / fftSize);
+ windowed[i] = frame[i] * w;
+ }
+
+ // FFT (using built-in)
+ const spectrum = realFFT(windowed);
+
+ // Convert to magnitude dB
+ const mag = new Float32Array(fftSize / 2);
+ for (let i = 0; i < fftSize / 2; ++i) {
+ const re = spectrum[i * 2];
+ const im = spectrum[i * 2 + 1];
+ const magLin = Math.sqrt(re * re + im * im);
+ mag[i] = 20 * Math.log10(Math.max(magLin, 1e-10));
+ }
+
+ // Find local maxima above threshold
+ const peaks = [];
+ for (let i = 2; i < mag.length - 2; ++i) {
+ if (mag[i] > thresholdDB &&
+ mag[i] > mag[i-1] && mag[i] > mag[i-2] &&
+ mag[i] > mag[i+1] && mag[i] > mag[i+2]) {
+
+ // Parabolic interpolation for sub-bin accuracy
+ const alpha = mag[i-1];
+ const beta = mag[i];
+ const gamma = mag[i+1];
+ const p = 0.5 * (alpha - gamma) / (alpha - 2*beta + gamma);
+
+ const binFreq = (i + p) * sampleRate / fftSize;
+ const ampDB = beta - 0.25 * (alpha - gamma) * p;
+ const ampLin = Math.pow(10, ampDB / 20);
+
+ peaks.push({freq: binFreq, amp: ampLin});
+ }
+ }
+
+ return peaks;
+}
+
+// Track partials across frames (birth/death/continuation)
+function trackPartials(frames, sampleRate) {
+ const partials = [];
+ const activePartials = [];
+ const trackingThreshold = 50; // Hz
+
+ for (const frame of frames) {
+ const matched = new Set();
+
+ // Match peaks to existing partials
+ for (const partial of activePartials) {
+ const lastFreq = partial.freqs[partial.freqs.length - 1];
+
+ let bestPeak = null;
+ let bestDist = Infinity;
+
+ for (let i = 0; i < frame.peaks.length; ++i) {
+ if (matched.has(i)) continue;
+
+ const peak = frame.peaks[i];
+ const dist = Math.abs(peak.freq - lastFreq);
+
+ if (dist < trackingThreshold && dist < bestDist) {
+ bestPeak = peak;
+ bestDist = dist;
+ partial.matchIdx = i;
+ }
+ }
+
+ if (bestPeak) {
+ // Continuation
+ partial.times.push(frame.time);
+ partial.freqs.push(bestPeak.freq);
+ partial.amps.push(bestPeak.amp);
+ partial.age = 0;
+ matched.add(partial.matchIdx);
+ } else {
+ // No match
+ partial.age++;
+ }
+ }
+
+ // Birth new partials from unmatched peaks
+ for (let i = 0; i < frame.peaks.length; ++i) {
+ if (matched.has(i)) continue;
+
+ const peak = frame.peaks[i];
+ activePartials.push({
+ times: [frame.time],
+ freqs: [peak.freq],
+ amps: [peak.amp],
+ age: 0,
+ matchIdx: -1
+ });
+ }
+
+ // Death old partials
+ for (let i = activePartials.length - 1; i >= 0; --i) {
+ if (activePartials[i].age > 2) {
+ // Move to finished if long enough
+ if (activePartials[i].times.length >= 4) {
+ partials.push(activePartials[i]);
+ }
+ activePartials.splice(i, 1);
+ }
+ }
+ }
+
+ // Finish remaining active partials
+ for (const partial of activePartials) {
+ if (partial.times.length >= 4) {
+ partials.push(partial);
+ }
+ }
+
+ return partials;
+}
+
+// Fit cubic bezier curve to trajectory
+function fitBezier(times, values) {
+ if (times.length < 4) {
+ // Not enough points, just use linear segments
+ return {
+ t0: times[0], v0: values[0],
+ t1: times[0], v1: values[0],
+ t2: times[times.length-1], v2: values[values.length-1],
+ t3: times[times.length-1], v3: values[values.length-1]
+ };
+ }
+
+ // Fix endpoints
+ const t0 = times[0];
+ const t3 = times[times.length - 1];
+ const v0 = values[0];
+ const v3 = values[values.length - 1];
+
+ // Solve for interior control points via least squares
+ // Simplification: place at 1/3 and 2/3 positions
+ const t1 = t0 + (t3 - t0) / 3;
+ const t2 = t0 + 2 * (t3 - t0) / 3;
+
+ // Find v1, v2 by evaluating at nearest data points
+ let v1 = v0, v2 = v3;
+ let minDist1 = Infinity, minDist2 = Infinity;
+
+ for (let i = 0; i < times.length; ++i) {
+ const dist1 = Math.abs(times[i] - t1);
+ const dist2 = Math.abs(times[i] - t2);
+
+ if (dist1 < minDist1) {
+ minDist1 = dist1;
+ v1 = values[i];
+ }
+ if (dist2 < minDist2) {
+ minDist2 = dist2;
+ v2 = values[i];
+ }
+ }
+
+ return {t0, v0, t1, v1, t2, v2, t3, v3};
+}