summaryrefslogtreecommitdiff
path: root/tools/mq_editor
diff options
context:
space:
mode:
authorskal <pascal.massimino@gmail.com>2026-02-17 20:54:15 +0100
committerskal <pascal.massimino@gmail.com>2026-02-17 20:54:15 +0100
commitcfcd238044c7ce06dfdf1f9e08c3842bfa07979b (patch)
tree12cd6da868c43af9c054bc44705326c1dae5f6de /tools/mq_editor
parentd151eb48b2c55d16a1d9caa6a7affb3e0793c3e7 (diff)
feat(mq_editor): Complete Phase 2 - JS synthesizer with STFT cache
Phase 2 - JS Synthesizer: - Created mq_synth.js with replica oscillator bank - Bezier curve evaluation (cubic De Casteljau algorithm) - Replica synthesis: frequency spread, amplitude decay, phase jitter - PCM buffer generation from extracted MQ partials - Normalization to prevent clipping - Key '1' plays synthesized audio, key '2' plays original - Playback comparison with animated playhead STFT Cache Optimization: - Created STFTCache class in fft.js for pre-computed windowed FFT frames - Clean interface: getFFT(t), getMagnitudeDB(t, freq), setHopSize() - Pre-computes all frames on WAV load (eliminates redundant FFT calls) - Dynamic cache update when hop size changes - Shared across spectrogram, tooltip, and mini-spectrum viewer - Significant performance improvement Mini-Spectrum Viewer: - Bottom-right overlay (200x100) matching spectral_editor style - Real-time FFT display at playhead or mouse position - 100-bar visualization with cyan-to-yellow gradient - Updates during playback or mouse hover Files: - tools/mq_editor/mq_synth.js (new) - tools/mq_editor/fft.js (STFTCache class added) - tools/mq_editor/index.html (synthesis playback, cache integration) - tools/mq_editor/viewer.js (cache-based rendering, spectrum viewer) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
Diffstat (limited to 'tools/mq_editor')
-rw-r--r--tools/mq_editor/README.md10
-rw-r--r--tools/mq_editor/fft.js86
-rw-r--r--tools/mq_editor/index.html91
-rw-r--r--tools/mq_editor/mq_synth.js107
-rw-r--r--tools/mq_editor/viewer.js113
5 files changed, 354 insertions, 53 deletions
diff --git a/tools/mq_editor/README.md b/tools/mq_editor/README.md
index 35e414c..1f43c19 100644
--- a/tools/mq_editor/README.md
+++ b/tools/mq_editor/README.md
@@ -41,8 +41,14 @@ open tools/mq_editor/index.html
- [x] Mouse tooltip (time/frequency/dB intensity)
- [x] Improved partial tracking (frequency-dependent threshold, candidate system)
- [x] Original WAV playback with animated playhead
- - [x] Keyboard shortcuts ('2' for playback)
-- [ ] Phase 2: JS synthesizer (preview playback)
+ - [x] Keyboard shortcuts ('1' synthesized, '2' original)
+ - [x] Mini-spectrum viewer (bottom-right overlay)
+ - [x] STFT cache for optimized FFT access
+- [x] Phase 2: JS synthesizer (preview playback)
+ - [x] Bezier curve evaluation (cubic)
+ - [x] Replica oscillator bank with frequency spread and jitter
+ - [x] PCM synthesis from extracted partials
+ - [x] Playback via Web Audio API
- [ ] Phase 3: Editing UI (drag control points, replicas)
- [ ] Phase 4: Export (.txt + C++ code generation)
diff --git a/tools/mq_editor/fft.js b/tools/mq_editor/fft.js
index 8610222..0668906 100644
--- a/tools/mq_editor/fft.js
+++ b/tools/mq_editor/fft.js
@@ -101,3 +101,89 @@ function realFFT(signal) {
return spectrum;
}
+
+// STFT Cache - Pre-computes and caches windowed FFT frames
+class STFTCache {
+ constructor(signal, sampleRate, fftSize, hopSize) {
+ this.signal = signal;
+ this.sampleRate = sampleRate;
+ this.fftSize = fftSize;
+ this.hopSize = hopSize;
+ this.frames = []; // Array of {time, offset, spectrum}
+ this.compute();
+ }
+
+ compute() {
+ this.frames = [];
+ const numFrames = Math.floor((this.signal.length - this.fftSize) / this.hopSize);
+
+ for (let frameIdx = 0; frameIdx < numFrames; ++frameIdx) {
+ const offset = frameIdx * this.hopSize;
+ const time = offset / this.sampleRate;
+
+ // Extract frame
+ const frame = this.signal.slice(offset, offset + this.fftSize);
+
+ // Apply Hann window
+ const windowed = new Float32Array(this.fftSize);
+ for (let i = 0; i < this.fftSize; ++i) {
+ const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / this.fftSize);
+ windowed[i] = frame[i] * w;
+ }
+
+ // Compute FFT
+ const spectrum = realFFT(windowed);
+
+ this.frames.push({time, offset, spectrum});
+ }
+ }
+
+ setHopSize(hopSize) {
+ if (hopSize === this.hopSize) return;
+ this.hopSize = hopSize;
+ this.compute();
+ }
+
+ setFFTSize(fftSize) {
+ if (fftSize === this.fftSize) return;
+ this.fftSize = fftSize;
+ this.compute();
+ }
+
+ getNumFrames() {
+ return this.frames.length;
+ }
+
+ getFrameAtIndex(frameIdx) {
+ if (frameIdx < 0 || frameIdx >= this.frames.length) return null;
+ return this.frames[frameIdx];
+ }
+
+ getFrameAtTime(t) {
+ if (this.frames.length === 0) return null;
+
+ // Find closest frame
+ const frameIdx = Math.floor(t * this.sampleRate / this.hopSize);
+ return this.getFrameAtIndex(frameIdx);
+ }
+
+ getFFT(t) {
+ const frame = this.getFrameAtTime(t);
+ return frame ? frame.spectrum : null;
+ }
+
+ // Get magnitude in dB at specific time and frequency
+ getMagnitudeDB(t, freq) {
+ const spectrum = this.getFFT(t);
+ if (!spectrum) return -80;
+
+ const bin = Math.round(freq * this.fftSize / this.sampleRate);
+ if (bin < 0 || bin >= this.fftSize / 2) return -80;
+
+ const re = spectrum[bin * 2];
+ const im = spectrum[bin * 2 + 1];
+ const mag = Math.sqrt(re * re + im * im);
+ return 20 * Math.log10(Math.max(mag, 1e-10));
+ }
+}
+
diff --git a/tools/mq_editor/index.html b/tools/mq_editor/index.html
index 1a07b61..84abd1c 100644
--- a/tools/mq_editor/index.html
+++ b/tools/mq_editor/index.html
@@ -86,7 +86,14 @@
</div>
</div>
- <canvas id="canvas" width="1400" height="600"></canvas>
+ <div style="position: relative;">
+ <canvas id="canvas" width="1400" height="600"></canvas>
+
+ <!-- Mini spectrum viewer (bottom-right overlay) -->
+ <div id="spectrumViewer" style="position: absolute; bottom: 10px; right: 10px; width: 200px; height: 100px; background: rgba(30, 30, 30, 0.9); border: 1px solid #555; border-radius: 3px; pointer-events: none;">
+ <canvas id="spectrumCanvas" width="200" height="100"></canvas>
+ </div>
+ </div>
<div id="tooltip" style="position: fixed; display: none; background: #2a2a2a; padding: 4px 8px; border: 1px solid #555; border-radius: 3px; pointer-events: none; font-size: 11px; z-index: 1000;"></div>
@@ -94,12 +101,15 @@
<script src="fft.js"></script>
<script src="mq_extract.js"></script>
+ <script src="mq_synth.js"></script>
<script src="viewer.js"></script>
<script>
let audioBuffer = null;
let viewer = null;
let audioContext = null;
let currentSource = null;
+ let extractedPartials = null;
+ let stftCache = null;
const wavFile = document.getElementById('wavFile');
const extractBtn = document.getElementById('extractBtn');
@@ -133,16 +143,36 @@
initAudioContext();
extractBtn.disabled = false;
playBtn.disabled = false;
- setStatus(`Loaded: ${audioBuffer.duration.toFixed(2)}s, ${audioBuffer.sampleRate}Hz, ${audioBuffer.numberOfChannels}ch`, 'info');
+ setStatus('Computing STFT cache...', 'info');
- // Create viewer
- viewer = new SpectrogramViewer(canvas, audioBuffer);
+ // Compute STFT cache
+ setTimeout(() => {
+ const signal = audioBuffer.getChannelData(0);
+ stftCache = new STFTCache(signal, audioBuffer.sampleRate, fftSize, parseInt(hopSize.value));
+
+ setStatus(`Loaded: ${audioBuffer.duration.toFixed(2)}s, ${audioBuffer.sampleRate}Hz, ${audioBuffer.numberOfChannels}ch (${stftCache.getNumFrames()} frames cached)`, 'info');
+
+ // Create viewer with cache
+ viewer = new SpectrogramViewer(canvas, audioBuffer, stftCache);
+ }, 10);
} catch (err) {
setStatus('Error loading WAV: ' + err.message, 'error');
console.error(err);
}
});
+ // Update cache when hop size changes
+ hopSize.addEventListener('change', () => {
+ if (stftCache) {
+ setStatus('Updating STFT cache...', 'info');
+ setTimeout(() => {
+ stftCache.setHopSize(parseInt(hopSize.value));
+ setStatus(`Cache updated (${stftCache.getNumFrames()} frames)`, 'info');
+ if (viewer) viewer.render();
+ }, 10);
+ }
+ });
+
// Extract partials
extractBtn.addEventListener('click', () => {
if (!audioBuffer) return;
@@ -161,6 +191,7 @@
const partials = extractPartials(audioBuffer, params);
+ extractedPartials = partials;
setStatus(`Extracted ${partials.length} partials`, 'info');
// Update viewer
@@ -235,12 +266,60 @@
status.className = type;
}
+ // Play synthesized audio
+ function playSynthesized() {
+ if (!extractedPartials || extractedPartials.length === 0) {
+ setStatus('No partials extracted yet', 'warn');
+ return;
+ }
+ if (!audioBuffer || !audioContext) return;
+
+ stopAudio();
+
+ setStatus('Synthesizing...', 'info');
+
+ // Synthesize PCM from partials
+ const sampleRate = audioBuffer.sampleRate;
+ const duration = audioBuffer.duration;
+ const pcm = synthesizeMQ(extractedPartials, sampleRate, duration);
+
+ // Create audio buffer
+ const synthBuffer = audioContext.createBuffer(1, pcm.length, sampleRate);
+ synthBuffer.getChannelData(0).set(pcm);
+
+ const startTime = audioContext.currentTime;
+ currentSource = audioContext.createBufferSource();
+ currentSource.buffer = synthBuffer;
+ currentSource.connect(audioContext.destination);
+ currentSource.start();
+
+ currentSource.onended = () => {
+ currentSource = null;
+ playBtn.disabled = false;
+ stopBtn.disabled = true;
+ viewer.setPlayheadTime(-1);
+ setStatus('Stopped', 'info');
+ };
+
+ playBtn.disabled = true;
+ stopBtn.disabled = false;
+ setStatus('Playing synthesized...', 'info');
+
+ // Animate playhead
+ function updatePlayhead() {
+ if (!currentSource) return;
+ const elapsed = audioContext.currentTime - startTime;
+ viewer.setPlayheadTime(elapsed);
+ requestAnimationFrame(updatePlayhead);
+ }
+ updatePlayhead();
+ }
+
// Keyboard shortcuts
document.addEventListener('keydown', (e) => {
if (e.code === 'Digit1') {
e.preventDefault();
- // TODO: Play synthesized (Phase 2)
- setStatus('Synthesized playback not yet implemented', 'warn');
+ playSynthesized();
} else if (e.code === 'Digit2') {
e.preventDefault();
if (!playBtn.disabled) {
diff --git a/tools/mq_editor/mq_synth.js b/tools/mq_editor/mq_synth.js
new file mode 100644
index 0000000..f1c7f73
--- /dev/null
+++ b/tools/mq_editor/mq_synth.js
@@ -0,0 +1,107 @@
+// MQ Synthesizer
+// Replica oscillator bank for sinusoidal synthesis
+
+// Evaluate cubic bezier curve at time t
+function evalBezier(curve, t) {
+ // Normalize t to [0, 1]
+ let u = (t - curve.t0) / (curve.t3 - curve.t0);
+ u = Math.max(0, Math.min(1, u));
+
+ // Cubic interpolation
+ const u1 = 1.0 - u;
+ return u1*u1*u1 * curve.v0 +
+ 3*u1*u1*u * curve.v1 +
+ 3*u1*u*u * curve.v2 +
+ u*u*u * curve.v3;
+}
+
+// Simple deterministic PRNG (for frequency spread and jitter)
+function randFloat(seed, min, max) {
+ // LCG parameters
+ const a = 1664525;
+ const c = 1013904223;
+ const m = 0x100000000; // 2^32
+
+ seed = (a * seed + c) % m;
+ const normalized = seed / m;
+ return min + normalized * (max - min);
+}
+
+// Synthesize audio from MQ partials
+// partials: array of {freqCurve, ampCurve, replicas: {offsets, decay_alpha, jitter, spread_above, spread_below}}
+// sampleRate: output sample rate (Hz)
+// duration: output duration (seconds)
+// Returns: Float32Array of PCM samples
+function synthesizeMQ(partials, sampleRate, duration) {
+ const numSamples = Math.floor(sampleRate * duration);
+ const pcm = new Float32Array(numSamples);
+
+ // Default replica config
+ const defaultReplicas = {
+ offsets: [1.0], // Just fundamental
+ decay_alpha: 0.1,
+ jitter: 0.05,
+ spread_above: 0.02,
+ spread_below: 0.02
+ };
+
+ for (let i = 0; i < numSamples; ++i) {
+ const t = i / sampleRate;
+ let sample = 0.0;
+
+ for (let p = 0; p < partials.length; ++p) {
+ const partial = partials[p];
+ const freqCurve = partial.freqCurve;
+ const ampCurve = partial.ampCurve;
+
+ // Skip if outside curve time range
+ if (t < freqCurve.t0 || t > freqCurve.t3) continue;
+
+ const f0 = evalBezier(freqCurve, t);
+ const A0 = evalBezier(ampCurve, t);
+
+ // Use default replicas if not specified
+ const replicas = partial.replicas || defaultReplicas;
+ const offsets = replicas.offsets || [1.0];
+ const decay_alpha = replicas.decay_alpha || 0.1;
+ const jitter = replicas.jitter || 0.05;
+ const spread_above = replicas.spread_above || 0.02;
+ const spread_below = replicas.spread_below || 0.02;
+
+ // For each replica offset
+ for (let r = 0; r < offsets.length; ++r) {
+ const ratio = offsets[r];
+
+ // Frequency spread (asymmetric randomization)
+ const seed1 = i * 12345 + p * 67890 + r;
+ const spread = randFloat(seed1, -spread_below, spread_above);
+ const f = f0 * ratio * (1.0 + spread);
+
+ // Amplitude decay
+ const A = A0 * Math.exp(-decay_alpha * Math.abs(f - f0));
+
+ // Phase with jitter
+ const seed2 = seed1 + 1;
+ const jitterPhase = randFloat(seed2, 0.0, 1.0) * jitter * 2.0 * Math.PI;
+ const phase = 2.0 * Math.PI * f * t + jitterPhase;
+
+ sample += A * Math.sin(phase);
+ }
+ }
+
+ pcm[i] = sample;
+ }
+
+ // Normalize to prevent clipping
+ let maxAbs = 0;
+ for (let i = 0; i < numSamples; ++i) {
+ maxAbs = Math.max(maxAbs, Math.abs(pcm[i]));
+ }
+ if (maxAbs > 1.0) {
+ for (let i = 0; i < numSamples; ++i) {
+ pcm[i] /= maxAbs;
+ }
+ }
+
+ return pcm;
+}
diff --git a/tools/mq_editor/viewer.js b/tools/mq_editor/viewer.js
index 459cd9e..451421d 100644
--- a/tools/mq_editor/viewer.js
+++ b/tools/mq_editor/viewer.js
@@ -2,10 +2,11 @@
// Handles all visualization: spectrogram, partials, zoom, mouse interaction
class SpectrogramViewer {
- constructor(canvas, audioBuffer) {
+ constructor(canvas, audioBuffer, stftCache) {
this.canvas = canvas;
this.ctx = canvas.getContext('2d');
this.audioBuffer = audioBuffer;
+ this.stftCache = stftCache;
this.partials = [];
// Fixed time bounds
@@ -30,6 +31,11 @@ class SpectrogramViewer {
// Playhead
this.playheadTime = -1; // -1 = not playing
+ // Spectrum viewer
+ this.spectrumCanvas = document.getElementById('spectrumCanvas');
+ this.spectrumCtx = this.spectrumCanvas ? this.spectrumCanvas.getContext('2d') : null;
+ this.spectrumTime = 0; // Time to display spectrum for
+
// Setup event handlers
this.setupMouseHandlers();
@@ -40,6 +46,9 @@ class SpectrogramViewer {
setPlayheadTime(time) {
this.playheadTime = time;
+ if (time >= 0) {
+ this.spectrumTime = time;
+ }
this.render();
}
@@ -88,6 +97,7 @@ class SpectrogramViewer {
this.renderPartials();
this.drawAxes();
this.drawPlayhead();
+ this.renderSpectrum();
}
drawPlayhead() {
@@ -108,46 +118,36 @@ class SpectrogramViewer {
// Render spectrogram background
renderSpectrogram() {
- const {canvas, ctx, audioBuffer} = this;
+ const {canvas, ctx, stftCache} = this;
const width = canvas.width;
const height = canvas.height;
ctx.fillStyle = '#000';
ctx.fillRect(0, 0, width, height);
- const signal = getMono(audioBuffer);
- const fftSize = 1024;
- const hopSize = 256;
- const sampleRate = audioBuffer.sampleRate;
+ if (!stftCache) return;
- const numFrames = Math.floor((signal.length - fftSize) / hopSize);
+ const sampleRate = this.audioBuffer.sampleRate;
+ const hopSize = stftCache.hopSize;
+ const fftSize = stftCache.fftSize;
const frameDuration = hopSize / sampleRate;
const viewDuration = this.t_view_max - this.t_view_min;
// Map view bounds to frame indices
const startFrameIdx = Math.floor(this.t_view_min * sampleRate / hopSize);
const endFrameIdx = Math.ceil(this.t_view_max * sampleRate / hopSize);
+ const numFrames = stftCache.getNumFrames();
for (let frameIdx = startFrameIdx; frameIdx < endFrameIdx; ++frameIdx) {
if (frameIdx < 0 || frameIdx >= numFrames) continue;
- const offset = frameIdx * hopSize;
- if (offset + fftSize > signal.length) break;
+ const frame = stftCache.getFrameAtIndex(frameIdx);
+ if (!frame) continue;
- const frame = signal.slice(offset, offset + fftSize);
-
- // Windowing
- const windowed = new Float32Array(fftSize);
- for (let i = 0; i < fftSize; ++i) {
- const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / fftSize);
- windowed[i] = frame[i] * w;
- }
-
- // FFT
- const spectrum = realFFT(windowed);
+ const spectrum = frame.spectrum;
// Compute frame time range
- const frameTime = frameIdx * hopSize / sampleRate;
+ const frameTime = frame.time;
const frameTimeEnd = frameTime + frameDuration;
const xStart = Math.floor((frameTime - this.t_view_min) / viewDuration * width);
@@ -350,6 +350,12 @@ class SpectrogramViewer {
const freq = this.canvasToFreq(y);
const intensity = this.getIntensityAt(time, freq);
+ // Update spectrum time when not playing
+ if (this.playheadTime < 0) {
+ this.spectrumTime = time;
+ this.renderSpectrum();
+ }
+
tooltip.style.left = (e.clientX + 10) + 'px';
tooltip.style.top = (e.clientY + 10) + 'px';
tooltip.style.display = 'block';
@@ -408,38 +414,55 @@ class SpectrogramViewer {
}
getIntensityAt(time, freq) {
- const signal = getMono(this.audioBuffer);
- const fftSize = 1024;
- const hopSize = 256;
- const sampleRate = this.audioBuffer.sampleRate;
+ if (!this.stftCache) return -80;
+ return this.stftCache.getMagnitudeDB(time, freq);
+ }
- const frameIdx = Math.floor(time * sampleRate / hopSize);
- const offset = frameIdx * hopSize;
+ renderSpectrum() {
+ if (!this.spectrumCtx || !this.stftCache) return;
- if (offset < 0 || offset + fftSize > signal.length) return -80;
+ const canvas = this.spectrumCanvas;
+ const ctx = this.spectrumCtx;
+ const width = canvas.width;
+ const height = canvas.height;
- const frame = signal.slice(offset, offset + fftSize);
+ // Clear
+ ctx.fillStyle = '#1e1e1e';
+ ctx.fillRect(0, 0, width, height);
- // Apply Hann window
- const windowed = new Float32Array(fftSize);
- for (let i = 0; i < fftSize; ++i) {
- const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / fftSize);
- windowed[i] = frame[i] * w;
- }
+ const spectrum = this.stftCache.getFFT(this.spectrumTime);
+ if (!spectrum) return;
- // FFT
- const spectrum = realFFT(windowed);
+ const fftSize = this.stftCache.fftSize;
- // Find closest bin
- const bin = Math.round(freq * fftSize / sampleRate);
- if (bin < 0 || bin >= fftSize / 2) return -80;
+ // Draw bars
+ const numBars = 100;
+ const barWidth = width / numBars;
+ const numBins = fftSize / 2;
- const re = spectrum[bin * 2];
- const im = spectrum[bin * 2 + 1];
- const mag = Math.sqrt(re * re + im * im);
- const magDB = 20 * Math.log10(Math.max(mag, 1e-10));
+ for (let i = 0; i < numBars; ++i) {
+ const binIdx = Math.floor(i * numBins / numBars);
+ const re = spectrum[binIdx * 2];
+ const im = spectrum[binIdx * 2 + 1];
+ const mag = Math.sqrt(re * re + im * im);
+ const magDB = 20 * Math.log10(Math.max(mag, 1e-10));
- return magDB;
+ // Normalize to [0, 1]
+ const normalized = (magDB + 80) / 80;
+ const clamped = Math.max(0, Math.min(1, normalized));
+ const intensity = Math.pow(clamped, 0.3);
+
+ const barHeight = intensity * height;
+ const x = i * barWidth;
+
+ // Gradient from bottom (cyan) to top (yellow)
+ const gradient = ctx.createLinearGradient(0, height - barHeight, 0, height);
+ gradient.addColorStop(0, '#4af');
+ gradient.addColorStop(1, '#fa4');
+
+ ctx.fillStyle = gradient;
+ ctx.fillRect(x, height - barHeight, barWidth - 1, barHeight);
+ }
}
// Utilities