diff options
| -rw-r--r-- | doc/SPECTRAL_BRUSH_2.md | 8 | ||||
| -rw-r--r-- | tools/mq_editor/README.md | 10 | ||||
| -rw-r--r-- | tools/mq_editor/fft.js | 86 | ||||
| -rw-r--r-- | tools/mq_editor/index.html | 91 | ||||
| -rw-r--r-- | tools/mq_editor/mq_synth.js | 107 | ||||
| -rw-r--r-- | tools/mq_editor/viewer.js | 113 |
6 files changed, 361 insertions, 54 deletions
diff --git a/doc/SPECTRAL_BRUSH_2.md b/doc/SPECTRAL_BRUSH_2.md index 442db6d..7714ce3 100644 --- a/doc/SPECTRAL_BRUSH_2.md +++ b/doc/SPECTRAL_BRUSH_2.md @@ -521,7 +521,13 @@ void pattern_callback(int sample_id, float volume) { - [x] Cubic bezier curve fitting for freq/amp trajectories - [x] Spectrogram visualization with zoom/scroll/playhead - [x] Original WAV playback -- [ ] Phase 2: JS synthesizer + - [x] STFT cache for optimized FFT computation + - [x] Mini-spectrum viewer +- [x] Phase 2: JS synthesizer + - [x] Bezier evaluation (cubic De Casteljau) + - [x] Replica oscillator bank (frequency spread, amplitude decay, phase jitter) + - [x] PCM synthesis from MQ partials + - [x] Playback comparison (original vs synthesized) - [ ] Phase 3: Web editor UI - [ ] Phase 4: C++ code generator - [ ] Phase 5: C++ synthesis + integration diff --git a/tools/mq_editor/README.md b/tools/mq_editor/README.md index 35e414c..1f43c19 100644 --- a/tools/mq_editor/README.md +++ b/tools/mq_editor/README.md @@ -41,8 +41,14 @@ open tools/mq_editor/index.html - [x] Mouse tooltip (time/frequency/dB intensity) - [x] Improved partial tracking (frequency-dependent threshold, candidate system) - [x] Original WAV playback with animated playhead - - [x] Keyboard shortcuts ('2' for playback) -- [ ] Phase 2: JS synthesizer (preview playback) + - [x] Keyboard shortcuts ('1' synthesized, '2' original) + - [x] Mini-spectrum viewer (bottom-right overlay) + - [x] STFT cache for optimized FFT access +- [x] Phase 2: JS synthesizer (preview playback) + - [x] Bezier curve evaluation (cubic) + - [x] Replica oscillator bank with frequency spread and jitter + - [x] PCM synthesis from extracted partials + - [x] Playback via Web Audio API - [ ] Phase 3: Editing UI (drag control points, replicas) - [ ] Phase 4: Export (.txt + C++ code generation) diff --git a/tools/mq_editor/fft.js b/tools/mq_editor/fft.js index 8610222..0668906 100644 --- a/tools/mq_editor/fft.js +++ b/tools/mq_editor/fft.js @@ -101,3 +101,89 @@ function realFFT(signal) { return spectrum; } + +// STFT Cache - Pre-computes and caches windowed FFT frames +class STFTCache { + constructor(signal, sampleRate, fftSize, hopSize) { + this.signal = signal; + this.sampleRate = sampleRate; + this.fftSize = fftSize; + this.hopSize = hopSize; + this.frames = []; // Array of {time, offset, spectrum} + this.compute(); + } + + compute() { + this.frames = []; + const numFrames = Math.floor((this.signal.length - this.fftSize) / this.hopSize); + + for (let frameIdx = 0; frameIdx < numFrames; ++frameIdx) { + const offset = frameIdx * this.hopSize; + const time = offset / this.sampleRate; + + // Extract frame + const frame = this.signal.slice(offset, offset + this.fftSize); + + // Apply Hann window + const windowed = new Float32Array(this.fftSize); + for (let i = 0; i < this.fftSize; ++i) { + const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / this.fftSize); + windowed[i] = frame[i] * w; + } + + // Compute FFT + const spectrum = realFFT(windowed); + + this.frames.push({time, offset, spectrum}); + } + } + + setHopSize(hopSize) { + if (hopSize === this.hopSize) return; + this.hopSize = hopSize; + this.compute(); + } + + setFFTSize(fftSize) { + if (fftSize === this.fftSize) return; + this.fftSize = fftSize; + this.compute(); + } + + getNumFrames() { + return this.frames.length; + } + + getFrameAtIndex(frameIdx) { + if (frameIdx < 0 || frameIdx >= this.frames.length) return null; + return this.frames[frameIdx]; + } + + getFrameAtTime(t) { + if (this.frames.length === 0) return null; + + // Find closest frame + const frameIdx = Math.floor(t * this.sampleRate / this.hopSize); + return this.getFrameAtIndex(frameIdx); + } + + getFFT(t) { + const frame = this.getFrameAtTime(t); + return frame ? frame.spectrum : null; + } + + // Get magnitude in dB at specific time and frequency + getMagnitudeDB(t, freq) { + const spectrum = this.getFFT(t); + if (!spectrum) return -80; + + const bin = Math.round(freq * this.fftSize / this.sampleRate); + if (bin < 0 || bin >= this.fftSize / 2) return -80; + + const re = spectrum[bin * 2]; + const im = spectrum[bin * 2 + 1]; + const mag = Math.sqrt(re * re + im * im); + return 20 * Math.log10(Math.max(mag, 1e-10)); + } +} + diff --git a/tools/mq_editor/index.html b/tools/mq_editor/index.html index 1a07b61..84abd1c 100644 --- a/tools/mq_editor/index.html +++ b/tools/mq_editor/index.html @@ -86,7 +86,14 @@ </div> </div> - <canvas id="canvas" width="1400" height="600"></canvas> + <div style="position: relative;"> + <canvas id="canvas" width="1400" height="600"></canvas> + + <!-- Mini spectrum viewer (bottom-right overlay) --> + <div id="spectrumViewer" style="position: absolute; bottom: 10px; right: 10px; width: 200px; height: 100px; background: rgba(30, 30, 30, 0.9); border: 1px solid #555; border-radius: 3px; pointer-events: none;"> + <canvas id="spectrumCanvas" width="200" height="100"></canvas> + </div> + </div> <div id="tooltip" style="position: fixed; display: none; background: #2a2a2a; padding: 4px 8px; border: 1px solid #555; border-radius: 3px; pointer-events: none; font-size: 11px; z-index: 1000;"></div> @@ -94,12 +101,15 @@ <script src="fft.js"></script> <script src="mq_extract.js"></script> + <script src="mq_synth.js"></script> <script src="viewer.js"></script> <script> let audioBuffer = null; let viewer = null; let audioContext = null; let currentSource = null; + let extractedPartials = null; + let stftCache = null; const wavFile = document.getElementById('wavFile'); const extractBtn = document.getElementById('extractBtn'); @@ -133,16 +143,36 @@ initAudioContext(); extractBtn.disabled = false; playBtn.disabled = false; - setStatus(`Loaded: ${audioBuffer.duration.toFixed(2)}s, ${audioBuffer.sampleRate}Hz, ${audioBuffer.numberOfChannels}ch`, 'info'); + setStatus('Computing STFT cache...', 'info'); - // Create viewer - viewer = new SpectrogramViewer(canvas, audioBuffer); + // Compute STFT cache + setTimeout(() => { + const signal = audioBuffer.getChannelData(0); + stftCache = new STFTCache(signal, audioBuffer.sampleRate, fftSize, parseInt(hopSize.value)); + + setStatus(`Loaded: ${audioBuffer.duration.toFixed(2)}s, ${audioBuffer.sampleRate}Hz, ${audioBuffer.numberOfChannels}ch (${stftCache.getNumFrames()} frames cached)`, 'info'); + + // Create viewer with cache + viewer = new SpectrogramViewer(canvas, audioBuffer, stftCache); + }, 10); } catch (err) { setStatus('Error loading WAV: ' + err.message, 'error'); console.error(err); } }); + // Update cache when hop size changes + hopSize.addEventListener('change', () => { + if (stftCache) { + setStatus('Updating STFT cache...', 'info'); + setTimeout(() => { + stftCache.setHopSize(parseInt(hopSize.value)); + setStatus(`Cache updated (${stftCache.getNumFrames()} frames)`, 'info'); + if (viewer) viewer.render(); + }, 10); + } + }); + // Extract partials extractBtn.addEventListener('click', () => { if (!audioBuffer) return; @@ -161,6 +191,7 @@ const partials = extractPartials(audioBuffer, params); + extractedPartials = partials; setStatus(`Extracted ${partials.length} partials`, 'info'); // Update viewer @@ -235,12 +266,60 @@ status.className = type; } + // Play synthesized audio + function playSynthesized() { + if (!extractedPartials || extractedPartials.length === 0) { + setStatus('No partials extracted yet', 'warn'); + return; + } + if (!audioBuffer || !audioContext) return; + + stopAudio(); + + setStatus('Synthesizing...', 'info'); + + // Synthesize PCM from partials + const sampleRate = audioBuffer.sampleRate; + const duration = audioBuffer.duration; + const pcm = synthesizeMQ(extractedPartials, sampleRate, duration); + + // Create audio buffer + const synthBuffer = audioContext.createBuffer(1, pcm.length, sampleRate); + synthBuffer.getChannelData(0).set(pcm); + + const startTime = audioContext.currentTime; + currentSource = audioContext.createBufferSource(); + currentSource.buffer = synthBuffer; + currentSource.connect(audioContext.destination); + currentSource.start(); + + currentSource.onended = () => { + currentSource = null; + playBtn.disabled = false; + stopBtn.disabled = true; + viewer.setPlayheadTime(-1); + setStatus('Stopped', 'info'); + }; + + playBtn.disabled = true; + stopBtn.disabled = false; + setStatus('Playing synthesized...', 'info'); + + // Animate playhead + function updatePlayhead() { + if (!currentSource) return; + const elapsed = audioContext.currentTime - startTime; + viewer.setPlayheadTime(elapsed); + requestAnimationFrame(updatePlayhead); + } + updatePlayhead(); + } + // Keyboard shortcuts document.addEventListener('keydown', (e) => { if (e.code === 'Digit1') { e.preventDefault(); - // TODO: Play synthesized (Phase 2) - setStatus('Synthesized playback not yet implemented', 'warn'); + playSynthesized(); } else if (e.code === 'Digit2') { e.preventDefault(); if (!playBtn.disabled) { diff --git a/tools/mq_editor/mq_synth.js b/tools/mq_editor/mq_synth.js new file mode 100644 index 0000000..f1c7f73 --- /dev/null +++ b/tools/mq_editor/mq_synth.js @@ -0,0 +1,107 @@ +// MQ Synthesizer +// Replica oscillator bank for sinusoidal synthesis + +// Evaluate cubic bezier curve at time t +function evalBezier(curve, t) { + // Normalize t to [0, 1] + let u = (t - curve.t0) / (curve.t3 - curve.t0); + u = Math.max(0, Math.min(1, u)); + + // Cubic interpolation + const u1 = 1.0 - u; + return u1*u1*u1 * curve.v0 + + 3*u1*u1*u * curve.v1 + + 3*u1*u*u * curve.v2 + + u*u*u * curve.v3; +} + +// Simple deterministic PRNG (for frequency spread and jitter) +function randFloat(seed, min, max) { + // LCG parameters + const a = 1664525; + const c = 1013904223; + const m = 0x100000000; // 2^32 + + seed = (a * seed + c) % m; + const normalized = seed / m; + return min + normalized * (max - min); +} + +// Synthesize audio from MQ partials +// partials: array of {freqCurve, ampCurve, replicas: {offsets, decay_alpha, jitter, spread_above, spread_below}} +// sampleRate: output sample rate (Hz) +// duration: output duration (seconds) +// Returns: Float32Array of PCM samples +function synthesizeMQ(partials, sampleRate, duration) { + const numSamples = Math.floor(sampleRate * duration); + const pcm = new Float32Array(numSamples); + + // Default replica config + const defaultReplicas = { + offsets: [1.0], // Just fundamental + decay_alpha: 0.1, + jitter: 0.05, + spread_above: 0.02, + spread_below: 0.02 + }; + + for (let i = 0; i < numSamples; ++i) { + const t = i / sampleRate; + let sample = 0.0; + + for (let p = 0; p < partials.length; ++p) { + const partial = partials[p]; + const freqCurve = partial.freqCurve; + const ampCurve = partial.ampCurve; + + // Skip if outside curve time range + if (t < freqCurve.t0 || t > freqCurve.t3) continue; + + const f0 = evalBezier(freqCurve, t); + const A0 = evalBezier(ampCurve, t); + + // Use default replicas if not specified + const replicas = partial.replicas || defaultReplicas; + const offsets = replicas.offsets || [1.0]; + const decay_alpha = replicas.decay_alpha || 0.1; + const jitter = replicas.jitter || 0.05; + const spread_above = replicas.spread_above || 0.02; + const spread_below = replicas.spread_below || 0.02; + + // For each replica offset + for (let r = 0; r < offsets.length; ++r) { + const ratio = offsets[r]; + + // Frequency spread (asymmetric randomization) + const seed1 = i * 12345 + p * 67890 + r; + const spread = randFloat(seed1, -spread_below, spread_above); + const f = f0 * ratio * (1.0 + spread); + + // Amplitude decay + const A = A0 * Math.exp(-decay_alpha * Math.abs(f - f0)); + + // Phase with jitter + const seed2 = seed1 + 1; + const jitterPhase = randFloat(seed2, 0.0, 1.0) * jitter * 2.0 * Math.PI; + const phase = 2.0 * Math.PI * f * t + jitterPhase; + + sample += A * Math.sin(phase); + } + } + + pcm[i] = sample; + } + + // Normalize to prevent clipping + let maxAbs = 0; + for (let i = 0; i < numSamples; ++i) { + maxAbs = Math.max(maxAbs, Math.abs(pcm[i])); + } + if (maxAbs > 1.0) { + for (let i = 0; i < numSamples; ++i) { + pcm[i] /= maxAbs; + } + } + + return pcm; +} diff --git a/tools/mq_editor/viewer.js b/tools/mq_editor/viewer.js index 459cd9e..451421d 100644 --- a/tools/mq_editor/viewer.js +++ b/tools/mq_editor/viewer.js @@ -2,10 +2,11 @@ // Handles all visualization: spectrogram, partials, zoom, mouse interaction class SpectrogramViewer { - constructor(canvas, audioBuffer) { + constructor(canvas, audioBuffer, stftCache) { this.canvas = canvas; this.ctx = canvas.getContext('2d'); this.audioBuffer = audioBuffer; + this.stftCache = stftCache; this.partials = []; // Fixed time bounds @@ -30,6 +31,11 @@ class SpectrogramViewer { // Playhead this.playheadTime = -1; // -1 = not playing + // Spectrum viewer + this.spectrumCanvas = document.getElementById('spectrumCanvas'); + this.spectrumCtx = this.spectrumCanvas ? this.spectrumCanvas.getContext('2d') : null; + this.spectrumTime = 0; // Time to display spectrum for + // Setup event handlers this.setupMouseHandlers(); @@ -40,6 +46,9 @@ class SpectrogramViewer { setPlayheadTime(time) { this.playheadTime = time; + if (time >= 0) { + this.spectrumTime = time; + } this.render(); } @@ -88,6 +97,7 @@ class SpectrogramViewer { this.renderPartials(); this.drawAxes(); this.drawPlayhead(); + this.renderSpectrum(); } drawPlayhead() { @@ -108,46 +118,36 @@ class SpectrogramViewer { // Render spectrogram background renderSpectrogram() { - const {canvas, ctx, audioBuffer} = this; + const {canvas, ctx, stftCache} = this; const width = canvas.width; const height = canvas.height; ctx.fillStyle = '#000'; ctx.fillRect(0, 0, width, height); - const signal = getMono(audioBuffer); - const fftSize = 1024; - const hopSize = 256; - const sampleRate = audioBuffer.sampleRate; + if (!stftCache) return; - const numFrames = Math.floor((signal.length - fftSize) / hopSize); + const sampleRate = this.audioBuffer.sampleRate; + const hopSize = stftCache.hopSize; + const fftSize = stftCache.fftSize; const frameDuration = hopSize / sampleRate; const viewDuration = this.t_view_max - this.t_view_min; // Map view bounds to frame indices const startFrameIdx = Math.floor(this.t_view_min * sampleRate / hopSize); const endFrameIdx = Math.ceil(this.t_view_max * sampleRate / hopSize); + const numFrames = stftCache.getNumFrames(); for (let frameIdx = startFrameIdx; frameIdx < endFrameIdx; ++frameIdx) { if (frameIdx < 0 || frameIdx >= numFrames) continue; - const offset = frameIdx * hopSize; - if (offset + fftSize > signal.length) break; + const frame = stftCache.getFrameAtIndex(frameIdx); + if (!frame) continue; - const frame = signal.slice(offset, offset + fftSize); - - // Windowing - const windowed = new Float32Array(fftSize); - for (let i = 0; i < fftSize; ++i) { - const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / fftSize); - windowed[i] = frame[i] * w; - } - - // FFT - const spectrum = realFFT(windowed); + const spectrum = frame.spectrum; // Compute frame time range - const frameTime = frameIdx * hopSize / sampleRate; + const frameTime = frame.time; const frameTimeEnd = frameTime + frameDuration; const xStart = Math.floor((frameTime - this.t_view_min) / viewDuration * width); @@ -350,6 +350,12 @@ class SpectrogramViewer { const freq = this.canvasToFreq(y); const intensity = this.getIntensityAt(time, freq); + // Update spectrum time when not playing + if (this.playheadTime < 0) { + this.spectrumTime = time; + this.renderSpectrum(); + } + tooltip.style.left = (e.clientX + 10) + 'px'; tooltip.style.top = (e.clientY + 10) + 'px'; tooltip.style.display = 'block'; @@ -408,38 +414,55 @@ class SpectrogramViewer { } getIntensityAt(time, freq) { - const signal = getMono(this.audioBuffer); - const fftSize = 1024; - const hopSize = 256; - const sampleRate = this.audioBuffer.sampleRate; + if (!this.stftCache) return -80; + return this.stftCache.getMagnitudeDB(time, freq); + } - const frameIdx = Math.floor(time * sampleRate / hopSize); - const offset = frameIdx * hopSize; + renderSpectrum() { + if (!this.spectrumCtx || !this.stftCache) return; - if (offset < 0 || offset + fftSize > signal.length) return -80; + const canvas = this.spectrumCanvas; + const ctx = this.spectrumCtx; + const width = canvas.width; + const height = canvas.height; - const frame = signal.slice(offset, offset + fftSize); + // Clear + ctx.fillStyle = '#1e1e1e'; + ctx.fillRect(0, 0, width, height); - // Apply Hann window - const windowed = new Float32Array(fftSize); - for (let i = 0; i < fftSize; ++i) { - const w = 0.5 - 0.5 * Math.cos(2 * Math.PI * i / fftSize); - windowed[i] = frame[i] * w; - } + const spectrum = this.stftCache.getFFT(this.spectrumTime); + if (!spectrum) return; - // FFT - const spectrum = realFFT(windowed); + const fftSize = this.stftCache.fftSize; - // Find closest bin - const bin = Math.round(freq * fftSize / sampleRate); - if (bin < 0 || bin >= fftSize / 2) return -80; + // Draw bars + const numBars = 100; + const barWidth = width / numBars; + const numBins = fftSize / 2; - const re = spectrum[bin * 2]; - const im = spectrum[bin * 2 + 1]; - const mag = Math.sqrt(re * re + im * im); - const magDB = 20 * Math.log10(Math.max(mag, 1e-10)); + for (let i = 0; i < numBars; ++i) { + const binIdx = Math.floor(i * numBins / numBars); + const re = spectrum[binIdx * 2]; + const im = spectrum[binIdx * 2 + 1]; + const mag = Math.sqrt(re * re + im * im); + const magDB = 20 * Math.log10(Math.max(mag, 1e-10)); - return magDB; + // Normalize to [0, 1] + const normalized = (magDB + 80) / 80; + const clamped = Math.max(0, Math.min(1, normalized)); + const intensity = Math.pow(clamped, 0.3); + + const barHeight = intensity * height; + const x = i * barWidth; + + // Gradient from bottom (cyan) to top (yellow) + const gradient = ctx.createLinearGradient(0, height - barHeight, 0, height); + gradient.addColorStop(0, '#4af'); + gradient.addColorStop(1, '#fa4'); + + ctx.fillStyle = gradient; + ctx.fillRect(x, height - barHeight, barWidth - 1, barHeight); + } } // Utilities |