diff --git a/build.ts b/build.ts index 53bcd6a..5c8d4a1 100644 --- a/build.ts +++ b/build.ts @@ -44,4 +44,19 @@ if (platformPkg) { } } +// Copy cavacore native library to dist +const cavacoreLib = platform === "darwin" + ? "libcavacore.dylib" + : platform === "win32" + ? "cavacore.dll" + : "libcavacore.so" +const cavacoreSrc = join("src", "native", cavacoreLib) + +if (existsSync(cavacoreSrc)) { + copyFileSync(cavacoreSrc, join("dist", cavacoreLib)) + console.log(`Copied cavacore library: ${cavacoreLib}`) +} else { + console.warn(`Warning: ${cavacoreSrc} not found — run scripts/build-cavacore.sh first`) +} + console.log("Build complete") diff --git a/package.json b/package.json index e7331cb..1cf3cff 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "scripts": { "start": "bun src/index.tsx", "dev": "bun --watch src/index.tsx", + "build:native": "bash scripts/build-cavacore.sh", "build": "bun run build.ts", "dist": "bun dist/index.js", "test": "bun test", diff --git a/scripts/build-cavacore.sh b/scripts/build-cavacore.sh new file mode 100755 index 0000000..b149f22 --- /dev/null +++ b/scripts/build-cavacore.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +# Build cavacore as a shared library with fftw3 statically linked. +# +# Prerequisites: +# macOS: brew install fftw +# Linux: apt install libfftw3-dev (or equivalent) +# +# Output: src/native/libcavacore.{dylib,so} + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SRC="$ROOT/cava/cavacore.c" +OUT_DIR="$ROOT/src/native" + +mkdir -p "$OUT_DIR" + +OS="$(uname -s)" +ARCH="$(uname -m)" + +# Resolve fftw3 paths +if [ "$OS" = "Darwin" ]; then + if [ "$ARCH" = "arm64" ]; then + FFTW_PREFIX="${FFTW_PREFIX:-/opt/homebrew}" + else + FFTW_PREFIX="${FFTW_PREFIX:-/usr/local}" + fi + LIB_EXT="dylib" + SHARED_FLAG="-dynamiclib" + INSTALL_NAME="-install_name @rpath/libcavacore.dylib" +else + FFTW_PREFIX="${FFTW_PREFIX:-/usr}" + LIB_EXT="so" + SHARED_FLAG="-shared" + INSTALL_NAME="" +fi + +FFTW_INCLUDE="$FFTW_PREFIX/include" +FFTW_STATIC="$FFTW_PREFIX/lib/libfftw3.a" + +if [ ! -f "$FFTW_STATIC" ]; then + echo "Error: libfftw3.a not found at $FFTW_STATIC" + echo "Install fftw3: brew install fftw (macOS) or apt install libfftw3-dev (Linux)" + exit 1 +fi + +if [ ! -f "$SRC" ]; then + echo "Error: cavacore.c not found at $SRC" + echo "Ensure the cava submodule is initialized: git submodule update --init" + exit 1 +fi + +OUT="$OUT_DIR/libcavacore.$LIB_EXT" + +echo "Building libcavacore.$LIB_EXT ($OS $ARCH)" +echo " Source: $SRC" +echo " FFTW3: $FFTW_STATIC" +echo " Output: $OUT" + +cc -O2 \ + $SHARED_FLAG \ + $INSTALL_NAME \ + -fPIC \ + -I"$FFTW_INCLUDE" \ + -I"$ROOT/cava" \ + -o "$OUT" \ + "$SRC" \ + "$FFTW_STATIC" \ + -lm + +echo "Built: $OUT" + +# Verify exported symbols +if [ "$OS" = "Darwin" ]; then + echo "" + echo "Exported symbols:" + nm -gU "$OUT" | grep "cava_" +fi diff --git a/src/native/libcavacore.dylib b/src/native/libcavacore.dylib new file mode 100755 index 0000000..f269f07 Binary files /dev/null and b/src/native/libcavacore.dylib differ diff --git a/tasks/real-time-audio-visualization/01-copy-cavacore-files.md b/tasks/real-time-audio-visualization/01-copy-cavacore-files.md new file mode 100644 index 0000000..ff58563 --- /dev/null +++ b/tasks/real-time-audio-visualization/01-copy-cavacore-files.md @@ -0,0 +1,57 @@ +# 01. Copy cavacore library files to project + +meta: + id: real-time-audio-visualization-01 + feature: real-time-audio-visualization + priority: P0 + depends_on: [] + tags: [setup, build] + +objective: +- Copy necessary cava library files from cava/ directory to src/utils/ for integration + +deliverables: +- src/utils/cavacore.h - Header file with cavacore API +- src/utils/cavacore.c - Implementation of cavacore library +- src/utils/audio-stream.h - Audio stream reader header +- src/utils/audio-stream.c - Audio stream reader implementation +- src/utils/audio-input.h - Common audio input types +- src/utils/audio-input.c - Audio input buffer management + +steps: +- Identify necessary files from cava/ directory: + - cavacore.h (API definition) + - cavacore.c (FFT processing implementation) + - input/common.h (common audio data structures) + - input/common.c (input buffer handling) + - input/fifo.h (FIFO input support - optional, for testing) + - input/fifo.c (FIFO input implementation - optional) +- Copy cavacore.h to src/utils/ +- Copy cavacore.c to src/utils/ +- Copy input/common.h to src/utils/ +- Copy input/common.c to src/utils/ +- Copy input/fifo.h to src/utils/ (optional) +- Copy input/fifo.c to src/utils/ (optional) +- Update file headers to indicate origin and licensing +- Note: Files from cava/ directory will be removed after integration + +tests: +- Unit: Verify all files compile successfully +- Integration: Ensure no import errors in TypeScript/JavaScript files +- Manual: Check that files are accessible from src/utils/ + +acceptance_criteria: +- All required cava files are copied to src/utils/ +- File headers include proper copyright and license information +- No compilation errors from missing dependencies +- Files are properly formatted for TypeScript/JavaScript integration + +validation: +- Run: `bun run build` to verify compilation +- Check: `ls src/utils/*.c src/utils/*.h` to confirm file presence + +notes: +- Only need cavacore.c, cavacore.h, and common.c/common.h for basic functionality +- input/fifo.c is optional - can be added later if needed +- FFTW library will need to be installed and linked separately +- The files will be integrated into the audio-waveform utility diff --git a/tasks/real-time-audio-visualization/02-integrate-cavacore-library.md b/tasks/real-time-audio-visualization/02-integrate-cavacore-library.md new file mode 100644 index 0000000..a18a1ec --- /dev/null +++ b/tasks/real-time-audio-visualization/02-integrate-cavacore-library.md @@ -0,0 +1,61 @@ +# 02. Integrate cavacore library for audio analysis + +meta: + id: real-time-audio-visualization-02 + feature: real-time-audio-visualization + priority: P0 + depends_on: [real-time-audio-visualization-01] + tags: [integration, audio-processing] + +objective: +- Create a TypeScript binding for the cavacore C library +- Provide async API for real-time audio frequency analysis + +deliverables: +- src/utils/cavacore.ts - TypeScript bindings for cavacore API +- src/utils/audio-visualizer.ts - High-level audio visualizer class +- Updated package.json with FFTW dependency + +steps: +- Review cavacore.h API and understand the interface: + - cava_init() - Initialize with parameters + - cava_execute() - Process samples and return frequencies + - cava_destroy() - Clean up +- Create cavacore.ts wrapper with TypeScript types: + - Define C-style structs as TypeScript interfaces + - Create bind() function to load shared library + - Implement async wrappers for init, execute, destroy +- Create audio-visualizer.ts class: + - Handle initialization with configurable parameters (bars, sensitivity, noise reduction) + - Provide execute() method that accepts audio samples and returns frequency data + - Manage cleanup and error handling +- Update package.json: + - Add @types/fftw3 dependency (if available) or document manual installation + - Add build instructions for linking FFTW library +- Test basic initialization and execution with dummy data + +tests: +- Unit: Test cavacore initialization with valid parameters +- Unit: Test cavacore execution with sample audio data +- Unit: Test cleanup and memory management +- Integration: Verify no memory leaks after multiple init/destroy cycles +- Integration: Test with actual audio data from ffmpeg + +acceptance_criteria: +- cavacore.ts compiles without TypeScript errors +- audio-visualizer.ts can be imported and initialized +- execute() method returns frequency data array +- Proper error handling for missing FFTW library +- No memory leaks in long-running tests + +validation: +- Run: `bun run build` to verify TypeScript compilation +- Run: `bun test` for unit tests +- Manual: Test with sample audio file and verify output + +notes: +- FFTW library needs to be installed separately on the system +- On macOS: brew install fftw +- On Linux: apt install libfftw3-dev +- The C code will need to be compiled into a shared library (.so/.dylib/.dll) +- For Bun, we can use `Bun.native()` or `Bun.ffi` to call C functions diff --git a/tasks/real-time-audio-visualization/03-create-audio-stream-reader.md b/tasks/real-time-audio-visualization/03-create-audio-stream-reader.md new file mode 100644 index 0000000..4acce28 --- /dev/null +++ b/tasks/real-time-audio-visualization/03-create-audio-stream-reader.md @@ -0,0 +1,72 @@ +# 03. Create audio stream reader for real-time data + +meta: + id: real-time-audio-visualization-03 + feature: real-time-audio-visualization + priority: P1 + depends_on: [real-time-audio-visualization-02] + tags: [audio-stream, real-time] + +objective: +- Create a mechanism to read audio stream from mpv backend +- Convert audio data to format suitable for cavacore processing +- Implement efficient buffer management + +deliverables: +- src/utils/audio-stream-reader.ts - Audio stream reader class +- src/utils/audio-stream-reader.test.ts - Unit tests + +steps: +- Design audio stream reader interface: + - Constructor accepts audio URL and backend (mpv) + - Start() method initiates audio playback and stream capture + - readSamples() method returns next batch of audio samples + - stop() method terminates stream capture +- Implement stream reading for mpv backend: + - Use mpv IPC to query audio device parameters (sample rate, channels) + - Use ffmpeg or similar to pipe audio output to stdin + - Read PCM samples from the stream +- Convert audio samples to appropriate format: + - Handle different bit depths (16-bit, 32-bit) + - Handle different sample rates (44100, 48000, etc.) + - Interleave stereo channels if needed +- Implement buffer management: + - Circular buffer for efficient sample storage + - Non-blocking read with timeout + - Sample rate conversion if needed +- Handle errors: + - Invalid audio URL + - Backend connection failure + - Sample format mismatch +- Create unit tests: + - Mock mpv backend + - Test sample reading + - Test buffer management + - Test error conditions + +tests: +- Unit: Test sample rate detection +- Unit: Test channel detection +- Unit: Test sample reading with valid data +- Unit: Test buffer overflow handling +- Unit: Test error handling for invalid audio +- Integration: Test with actual audio file and mpv +- Integration: Test with ffplay backend + +acceptance_criteria: +- Audio stream reader successfully reads audio data from mpv +- Samples are converted to 16-bit PCM format +- Buffer management prevents overflow +- Error handling works for invalid audio +- No memory leaks in long-running tests + +validation: +- Run: `bun test` for unit tests +- Manual: Play audio and verify stream reader captures data +- Manual: Test with different audio formats (mp3, wav, m4a) + +notes: +- mpv can output audio via pipe to stdin using --audio-file-pipe +- Alternative: Use ffmpeg to re-encode audio to standard format +- Sample rate conversion may be needed for cavacore compatibility +- For simplicity, start with 16-bit PCM, single channel (mono) diff --git a/tasks/real-time-audio-visualization/04-create-realtime-waveform-component.md b/tasks/real-time-audio-visualization/04-create-realtime-waveform-component.md new file mode 100644 index 0000000..1bdd4b9 --- /dev/null +++ b/tasks/real-time-audio-visualization/04-create-realtime-waveform-component.md @@ -0,0 +1,75 @@ +# 04. Create realtime waveform component + +meta: + id: real-time-audio-visualization-04 + feature: real-time-audio-visualization + priority: P1 + depends_on: [real-time-audio-visualization-03] + tags: [component, ui] + +objective: +- Create a SolidJS component that displays real-time audio visualization +- Integrate audio-visualizer and audio-stream-reader +- Display frequency data as visual waveform bars + +deliverables: +- src/components/RealtimeWaveform.tsx - Real-time waveform component +- src/components/RealtimeWaveform.test.tsx - Component tests + +steps: +- Create RealtimeWaveform component: + - Accept props: audioUrl, position, duration, isPlaying, onSeek, resolution + - Initialize audio-visualizer with cavacore + - Initialize audio-stream-reader for mpv backend + - Create render loop that: + - Reads audio samples from stream reader + - Passes samples to cavacore execute() + - Gets frequency data back + - Maps frequency data to visual bars + - Renders bars with appropriate colors +- Implement rendering logic: + - Map frequency values to bar heights + - Color-code bars based on intensity + - Handle played vs unplayed portions + - Support click-to-seek +- Create visual style: + - Use terminal block characters for bars + - Apply colors based on frequency bands (bass, mid, treble) + - Add visual flair (gradients, glow effects if possible) +- Implement state management: + - Track current frequency data + - Track playback position + - Handle component lifecycle (cleanup) +- Create unit tests: + - Test component initialization + - Test render loop + - Test click-to-seek + - Test cleanup + +tests: +- Unit: Test component props +- Unit: Test frequency data mapping +- Unit: Test visual bar rendering +- Integration: Test with mock audio data +- Integration: Test with actual audio playback + +acceptance_criteria: +- Component renders without errors +- Visual bars update in real-time during playback +- Frequency data is correctly calculated from audio samples +- Click-to-seek works +- Component cleans up resources properly +- Visual style matches design requirements + +validation: +- Run: `bun test` for unit tests +- Manual: Play audio and verify visualization updates +- Manual: Test seeking and verify visualization follows +- Performance: Monitor frame rate and CPU usage + +notes: +- Use SolidJS createEffect for reactive updates +- Keep render loop efficient to maintain 60fps +- Consider debouncing if processing is too heavy +- May need to adjust sample rate for performance +- Visual style should complement existing MergedWaveform design diff --git a/tasks/real-time-audio-visualization/05-update-player-visualization.md b/tasks/real-time-audio-visualization/05-update-player-visualization.md new file mode 100644 index 0000000..44e0cff --- /dev/null +++ b/tasks/real-time-audio-visualization/05-update-player-visualization.md @@ -0,0 +1,64 @@ +# 05. Update Player component to use realtime visualization + +meta: + id: real-time-audio-visualization-05 + feature: real-time-audio-visualization + priority: P1 + depends_on: [real-time-audio-visualization-04] + tags: [integration, player] + +objective: +- Replace static waveform display with real-time visualization +- Update Player.tsx to use RealtimeWaveform component +- Ensure seamless transition and proper state management + +deliverables: +- Updated src/components/Player.tsx +- Updated src/components/MergedWaveform.tsx (optional, for fallback) +- Documentation of changes + +steps: +- Update Player.tsx: + - Import RealtimeWaveform component + - Replace MergedWaveform with RealtimeWaveform + - Pass same props (audioUrl, position, duration, isPlaying, onSeek) + - Remove audioUrl from props if no longer needed +- Test with different audio formats +- Add fallback handling: + - If realtime visualization fails, show static waveform + - Graceful degradation for systems without FFTW +- Update component documentation +- Test all player controls work with new visualization +- Verify keyboard shortcuts still work +- Test seek, pause, resume, volume, speed controls + +tests: +- Unit: Test Player with RealtimeWaveform +- Integration: Test complete playback flow +- Integration: Test seek functionality +- Integration: Test pause/resume +- Integration: Test volume and speed changes +- Integration: Test with different audio formats +- Manual: Verify all player features work correctly + +acceptance_criteria: +- Player displays real-time visualization during playback +- All player controls work correctly +- Seek functionality works with visualization +- Graceful fallback for systems without FFTW +- No regression in existing functionality +- Visual style matches design requirements + +validation: +- Run: `bun run build` to verify compilation +- Run: `bun test` for integration tests +- Manual: Play various audio files +- Manual: Test all keyboard shortcuts +- Performance: Monitor frame rate and CPU usage + +notes: +- Keep MergedWaveform as fallback option +- Consider showing a loading state while visualizer initializes +- May need to handle the case where mpv doesn't support audio pipe +- The visualizer should integrate smoothly with existing Player layout +- Consider adding a toggle to switch between static and realtime visualization diff --git a/tasks/real-time-audio-visualization/06-add-visualizer-controls.md b/tasks/real-time-audio-visualization/06-add-visualizer-controls.md new file mode 100644 index 0000000..03895d7 --- /dev/null +++ b/tasks/real-time-audio-visualization/06-add-visualizer-controls.md @@ -0,0 +1,78 @@ +# 06. Add visualizer controls and settings + +meta: + id: real-time-audio-visualization-06 + feature: real-time-audio-visualization + priority: P2 + depends_on: [real-time-audio-visualization-05] + tags: [ui, controls, settings] + +objective: +- Add user controls for visualizer settings +- Create settings panel for customization +- Allow users to adjust visualizer parameters + +deliverables: +- src/components/VisualizerSettings.tsx - Settings component +- Updated src/components/Player.tsx - Settings panel integration +- src/types/settings.ts - Visualizer settings type definition +- src/stores/settings.ts - Settings state management + +steps: +- Define visualizer settings types: + - Number of bars (resolution) + - Sensitivity (autosens toggle + manual value) + - Noise reduction level + - Frequency cutoffs (low/high) + - Bar width and spacing + - Color scheme options +- Create VisualizerSettings component: + - Display current settings + - Allow adjusting each parameter + - Show real-time feedback + - Save settings to app store +- Integrate with Player component: + - Add settings button + - Show settings panel when toggled + - Apply settings to RealtimeWaveform component +- Update settings state management: + - Load saved settings from app store + - Save settings on change + - Provide default values +- Create UI for settings: + - Keyboard shortcuts for quick adjustment + - Visual feedback for changes + - Help text for each setting +- Add settings persistence +- Create tests for settings component + +tests: +- Unit: Test settings type definitions +- Unit: Test settings state management +- Unit: Test VisualizerSettings component +- Integration: Test settings apply to visualization +- Integration: Test settings persistence +- Manual: Test all settings controls + +acceptance_criteria: +- VisualizerSettings component renders correctly +- All settings can be adjusted +- Changes apply in real-time +- Settings persist between sessions +- Keyboard shortcuts work +- Component handles invalid settings gracefully + +validation: +- Run: `bun test` for unit tests +- Run: `bun test` for integration tests +- Manual: Test all settings +- Manual: Test keyboard shortcuts +- Manual: Test settings persistence + +notes: +- Settings should have sensible defaults +- Some settings may require visualizer re-initialization +- Consider limiting certain settings to avoid performance issues +- Add tooltips or help text for complex settings +- Settings should be optional - users can start without them +- Keep UI simple and intuitive diff --git a/tasks/real-time-audio-visualization/README.md b/tasks/real-time-audio-visualization/README.md new file mode 100644 index 0000000..3991bf0 --- /dev/null +++ b/tasks/real-time-audio-visualization/README.md @@ -0,0 +1,30 @@ +# Real-time Audio Visualization + +Objective: Integrate cava library for real-time audio visualization in Player component + +Status legend: [ ] todo, [~] in-progress, [x] done + +Tasks +- [x] 01 — Copy cavacore library files to project → `01-copy-cavacore-files.md` +- [ ] 02 — Integrate cavacore library for audio analysis → `02-integrate-cavacore-library.md` +- [ ] 03 — Create audio stream reader for real-time data → `03-create-audio-stream-reader.md` +- [ ] 04 — Create realtime waveform component → `04-create-realtime-waveform-component.md` +- [ ] 05 — Update Player component to use realtime visualization → `05-update-player-visualization.md` +- [ ] 06 — Add visualizer controls and settings → `06-add-visualizer-controls.md` + +Dependencies +- 01 depends on (none) +- 02 depends on 01 +- 03 depends on 02 +- 04 depends on 03 +- 05 depends on 04 +- 06 depends on 05 + +Exit criteria +- Audio visualization updates in real-time during playback +- Waveform bars respond to actual audio frequencies +- Visualizer controls (sensitivity, bar count) work +- Performance is smooth with 60fps updates +- All necessary cava files are integrated into project + +Note: Files from cava/ directory will be removed after integration diff --git a/tests/cavacore-smoke.ts b/tests/cavacore-smoke.ts new file mode 100644 index 0000000..9827201 --- /dev/null +++ b/tests/cavacore-smoke.ts @@ -0,0 +1,58 @@ +/** + * Smoke test: load libcavacore.dylib via bun:ffi, init → execute → destroy. + * Run: bun tests/cavacore-smoke.ts + */ +import { dlopen, FFIType, ptr } from "bun:ffi" +import { join } from "path" + +const libPath = join(import.meta.dir, "..", "src", "native", "libcavacore.dylib") + +const lib = dlopen(libPath, { + cava_init: { + args: [FFIType.i32, FFIType.u32, FFIType.i32, FFIType.i32, FFIType.double, FFIType.i32, FFIType.i32], + returns: FFIType.ptr, + }, + cava_execute: { + args: [FFIType.ptr, FFIType.i32, FFIType.ptr, FFIType.ptr], + returns: FFIType.void, + }, + cava_destroy: { + args: [FFIType.ptr], + returns: FFIType.void, + }, +}) + +const bars = 10 +const rate = 44100 +const channels = 1 + +// Init +const plan = lib.symbols.cava_init(bars, rate, channels, 1, 0.77, 50, 10000) +if (!plan) { + console.error("FAIL: cava_init returned null") + process.exit(1) +} +console.log("cava_init OK, plan pointer:", plan) + +// Generate a 200Hz sine wave test signal +const bufferSize = 512 +const cavaIn = new Float64Array(bufferSize) +const cavaOut = new Float64Array(bars * channels) + +for (let k = 0; k < 100; k++) { + for (let n = 0; n < bufferSize; n++) { + cavaIn[n] = Math.sin(2 * Math.PI * 200 / rate * (n + k * bufferSize)) * 20000 + } + lib.symbols.cava_execute(ptr(cavaIn), bufferSize, ptr(cavaOut), plan) +} + +console.log("cava_execute OK, output:", Array.from(cavaOut).map(v => v.toFixed(3))) + +// Check that bar 2 (200Hz) has the peak +const maxIdx = cavaOut.indexOf(Math.max(...cavaOut)) +console.log(`Peak at bar ${maxIdx} (expected ~2 for 200Hz)`) + +// Destroy +lib.symbols.cava_destroy(plan) +console.log("cava_destroy OK") +console.log("\nSMOKE TEST PASSED")