implementing cava for real time visualization

build.ts

@@ -44,4 +44,19 @@ if (platformPkg) {
   }
 }
 
+// Copy cavacore native library to dist
+const cavacoreLib = platform === "darwin"
+  ? "libcavacore.dylib"
+  : platform === "win32"
+    ? "cavacore.dll"
+    : "libcavacore.so"
+const cavacoreSrc = join("src", "native", cavacoreLib)
+
+if (existsSync(cavacoreSrc)) {
+  copyFileSync(cavacoreSrc, join("dist", cavacoreLib))
+  console.log(`Copied cavacore library: ${cavacoreLib}`)
+} else {
+  console.warn(`Warning: ${cavacoreSrc} not found — run scripts/build-cavacore.sh first`)
+}
+
 console.log("Build complete")

package.json

@@ -9,6 +9,7 @@
   "scripts": {
     "start": "bun src/index.tsx",
     "dev": "bun --watch src/index.tsx",
+    "build:native": "bash scripts/build-cavacore.sh",
     "build": "bun run build.ts",
     "dist": "bun dist/index.js",
     "test": "bun test",

scripts/build-cavacore.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+#
+# Build cavacore as a shared library with fftw3 statically linked.
+#
+# Prerequisites:
+#   macOS:  brew install fftw
+#   Linux:  apt install libfftw3-dev  (or equivalent)
+#
+# Output: src/native/libcavacore.{dylib,so}
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+SRC="$ROOT/cava/cavacore.c"
+OUT_DIR="$ROOT/src/native"
+
+mkdir -p "$OUT_DIR"
+
+OS="$(uname -s)"
+ARCH="$(uname -m)"
+
+# Resolve fftw3 paths
+if [ "$OS" = "Darwin" ]; then
+  if [ "$ARCH" = "arm64" ]; then
+    FFTW_PREFIX="${FFTW_PREFIX:-/opt/homebrew}"
+  else
+    FFTW_PREFIX="${FFTW_PREFIX:-/usr/local}"
+  fi
+  LIB_EXT="dylib"
+  SHARED_FLAG="-dynamiclib"
+  INSTALL_NAME="-install_name @rpath/libcavacore.dylib"
+else
+  FFTW_PREFIX="${FFTW_PREFIX:-/usr}"
+  LIB_EXT="so"
+  SHARED_FLAG="-shared"
+  INSTALL_NAME=""
+fi
+
+FFTW_INCLUDE="$FFTW_PREFIX/include"
+FFTW_STATIC="$FFTW_PREFIX/lib/libfftw3.a"
+
+if [ ! -f "$FFTW_STATIC" ]; then
+  echo "Error: libfftw3.a not found at $FFTW_STATIC"
+  echo "Install fftw3: brew install fftw (macOS) or apt install libfftw3-dev (Linux)"
+  exit 1
+fi
+
+if [ ! -f "$SRC" ]; then
+  echo "Error: cavacore.c not found at $SRC"
+  echo "Ensure the cava submodule is initialized: git submodule update --init"
+  exit 1
+fi
+
+OUT="$OUT_DIR/libcavacore.$LIB_EXT"
+
+echo "Building libcavacore.$LIB_EXT ($OS $ARCH)"
+echo "  Source:  $SRC"
+echo "  FFTW3:   $FFTW_STATIC"
+echo "  Output:  $OUT"
+
+cc -O2 \
+  $SHARED_FLAG \
+  $INSTALL_NAME \
+  -fPIC \
+  -I"$FFTW_INCLUDE" \
+  -I"$ROOT/cava" \
+  -o "$OUT" \
+  "$SRC" \
+  "$FFTW_STATIC" \
+  -lm
+
+echo "Built: $OUT"
+
+# Verify exported symbols
+if [ "$OS" = "Darwin" ]; then
+  echo ""
+  echo "Exported symbols:"
+  nm -gU "$OUT" | grep "cava_"
+fi

tasks/real-time-audio-visualization/01-copy-cavacore-files.md

@@ -0,0 +1,57 @@
+# 01. Copy cavacore library files to project
+
+meta:
+  id: real-time-audio-visualization-01
+  feature: real-time-audio-visualization
+  priority: P0
+  depends_on: []
+  tags: [setup, build]
+
+objective:
+- Copy necessary cava library files from cava/ directory to src/utils/ for integration
+
+deliverables:
+- src/utils/cavacore.h - Header file with cavacore API
+- src/utils/cavacore.c - Implementation of cavacore library
+- src/utils/audio-stream.h - Audio stream reader header
+- src/utils/audio-stream.c - Audio stream reader implementation
+- src/utils/audio-input.h - Common audio input types
+- src/utils/audio-input.c - Audio input buffer management
+
+steps:
+- Identify necessary files from cava/ directory:
+  - cavacore.h (API definition)
+  - cavacore.c (FFT processing implementation)
+  - input/common.h (common audio data structures)
+  - input/common.c (input buffer handling)
+  - input/fifo.h (FIFO input support - optional, for testing)
+  - input/fifo.c (FIFO input implementation - optional)
+- Copy cavacore.h to src/utils/
+- Copy cavacore.c to src/utils/
+- Copy input/common.h to src/utils/
+- Copy input/common.c to src/utils/
+- Copy input/fifo.h to src/utils/ (optional)
+- Copy input/fifo.c to src/utils/ (optional)
+- Update file headers to indicate origin and licensing
+- Note: Files from cava/ directory will be removed after integration
+
+tests:
+- Unit: Verify all files compile successfully
+- Integration: Ensure no import errors in TypeScript/JavaScript files
+- Manual: Check that files are accessible from src/utils/
+
+acceptance_criteria:
+- All required cava files are copied to src/utils/
+- File headers include proper copyright and license information
+- No compilation errors from missing dependencies
+- Files are properly formatted for TypeScript/JavaScript integration
+
+validation:
+- Run: `bun run build` to verify compilation
+- Check: `ls src/utils/*.c src/utils/*.h` to confirm file presence
+
+notes:
+- Only need cavacore.c, cavacore.h, and common.c/common.h for basic functionality
+- input/fifo.c is optional - can be added later if needed
+- FFTW library will need to be installed and linked separately
+- The files will be integrated into the audio-waveform utility

tasks/real-time-audio-visualization/02-integrate-cavacore-library.md

@@ -0,0 +1,61 @@
+# 02. Integrate cavacore library for audio analysis
+
+meta:
+  id: real-time-audio-visualization-02
+  feature: real-time-audio-visualization
+  priority: P0
+  depends_on: [real-time-audio-visualization-01]
+  tags: [integration, audio-processing]
+
+objective:
+- Create a TypeScript binding for the cavacore C library
+- Provide async API for real-time audio frequency analysis
+
+deliverables:
+- src/utils/cavacore.ts - TypeScript bindings for cavacore API
+- src/utils/audio-visualizer.ts - High-level audio visualizer class
+- Updated package.json with FFTW dependency
+
+steps:
+- Review cavacore.h API and understand the interface:
+  - cava_init() - Initialize with parameters
+  - cava_execute() - Process samples and return frequencies
+  - cava_destroy() - Clean up
+- Create cavacore.ts wrapper with TypeScript types:
+  - Define C-style structs as TypeScript interfaces
+  - Create bind() function to load shared library
+  - Implement async wrappers for init, execute, destroy
+- Create audio-visualizer.ts class:
+  - Handle initialization with configurable parameters (bars, sensitivity, noise reduction)
+  - Provide execute() method that accepts audio samples and returns frequency data
+  - Manage cleanup and error handling
+- Update package.json:
+  - Add @types/fftw3 dependency (if available) or document manual installation
+  - Add build instructions for linking FFTW library
+- Test basic initialization and execution with dummy data
+
+tests:
+- Unit: Test cavacore initialization with valid parameters
+- Unit: Test cavacore execution with sample audio data
+- Unit: Test cleanup and memory management
+- Integration: Verify no memory leaks after multiple init/destroy cycles
+- Integration: Test with actual audio data from ffmpeg
+
+acceptance_criteria:
+- cavacore.ts compiles without TypeScript errors
+- audio-visualizer.ts can be imported and initialized
+- execute() method returns frequency data array
+- Proper error handling for missing FFTW library
+- No memory leaks in long-running tests
+
+validation:
+- Run: `bun run build` to verify TypeScript compilation
+- Run: `bun test` for unit tests
+- Manual: Test with sample audio file and verify output
+
+notes:
+- FFTW library needs to be installed separately on the system
+- On macOS: brew install fftw
+- On Linux: apt install libfftw3-dev
+- The C code will need to be compiled into a shared library (.so/.dylib/.dll)
+- For Bun, we can use `Bun.native()` or `Bun.ffi` to call C functions

tasks/real-time-audio-visualization/03-create-audio-stream-reader.md

@@ -0,0 +1,72 @@
+# 03. Create audio stream reader for real-time data
+
+meta:
+  id: real-time-audio-visualization-03
+  feature: real-time-audio-visualization
+  priority: P1
+  depends_on: [real-time-audio-visualization-02]
+  tags: [audio-stream, real-time]
+
+objective:
+- Create a mechanism to read audio stream from mpv backend
+- Convert audio data to format suitable for cavacore processing
+- Implement efficient buffer management
+
+deliverables:
+- src/utils/audio-stream-reader.ts - Audio stream reader class
+- src/utils/audio-stream-reader.test.ts - Unit tests
+
+steps:
+- Design audio stream reader interface:
+  - Constructor accepts audio URL and backend (mpv)
+  - Start() method initiates audio playback and stream capture
+  - readSamples() method returns next batch of audio samples
+  - stop() method terminates stream capture
+- Implement stream reading for mpv backend:
+  - Use mpv IPC to query audio device parameters (sample rate, channels)
+  - Use ffmpeg or similar to pipe audio output to stdin
+  - Read PCM samples from the stream
+- Convert audio samples to appropriate format:
+  - Handle different bit depths (16-bit, 32-bit)
+  - Handle different sample rates (44100, 48000, etc.)
+  - Interleave stereo channels if needed
+- Implement buffer management:
+  - Circular buffer for efficient sample storage
+  - Non-blocking read with timeout
+  - Sample rate conversion if needed
+- Handle errors:
+  - Invalid audio URL
+  - Backend connection failure
+  - Sample format mismatch
+- Create unit tests:
+  - Mock mpv backend
+  - Test sample reading
+  - Test buffer management
+  - Test error conditions
+
+tests:
+- Unit: Test sample rate detection
+- Unit: Test channel detection
+- Unit: Test sample reading with valid data
+- Unit: Test buffer overflow handling
+- Unit: Test error handling for invalid audio
+- Integration: Test with actual audio file and mpv
+- Integration: Test with ffplay backend
+
+acceptance_criteria:
+- Audio stream reader successfully reads audio data from mpv
+- Samples are converted to 16-bit PCM format
+- Buffer management prevents overflow
+- Error handling works for invalid audio
+- No memory leaks in long-running tests
+
+validation:
+- Run: `bun test` for unit tests
+- Manual: Play audio and verify stream reader captures data
+- Manual: Test with different audio formats (mp3, wav, m4a)
+
+notes:
+- mpv can output audio via pipe to stdin using --audio-file-pipe
+- Alternative: Use ffmpeg to re-encode audio to standard format
+- Sample rate conversion may be needed for cavacore compatibility
+- For simplicity, start with 16-bit PCM, single channel (mono)

tasks/real-time-audio-visualization/04-create-realtime-waveform-component.md

@@ -0,0 +1,75 @@
+# 04. Create realtime waveform component
+
+meta:
+  id: real-time-audio-visualization-04
+  feature: real-time-audio-visualization
+  priority: P1
+  depends_on: [real-time-audio-visualization-03]
+  tags: [component, ui]
+
+objective:
+- Create a SolidJS component that displays real-time audio visualization
+- Integrate audio-visualizer and audio-stream-reader
+- Display frequency data as visual waveform bars
+
+deliverables:
+- src/components/RealtimeWaveform.tsx - Real-time waveform component
+- src/components/RealtimeWaveform.test.tsx - Component tests
+
+steps:
+- Create RealtimeWaveform component:
+  - Accept props: audioUrl, position, duration, isPlaying, onSeek, resolution
+  - Initialize audio-visualizer with cavacore
+  - Initialize audio-stream-reader for mpv backend
+  - Create render loop that:
+    - Reads audio samples from stream reader
+    - Passes samples to cavacore execute()
+    - Gets frequency data back
+    - Maps frequency data to visual bars
+    - Renders bars with appropriate colors
+- Implement rendering logic:
+  - Map frequency values to bar heights
+  - Color-code bars based on intensity
+  - Handle played vs unplayed portions
+  - Support click-to-seek
+- Create visual style:
+  - Use terminal block characters for bars
+  - Apply colors based on frequency bands (bass, mid, treble)
+  - Add visual flair (gradients, glow effects if possible)
+- Implement state management:
+  - Track current frequency data
+  - Track playback position
+  - Handle component lifecycle (cleanup)
+- Create unit tests:
+  - Test component initialization
+  - Test render loop
+  - Test click-to-seek
+  - Test cleanup
+
+tests:
+- Unit: Test component props
+- Unit: Test frequency data mapping
+- Unit: Test visual bar rendering
+- Integration: Test with mock audio data
+- Integration: Test with actual audio playback
+
+acceptance_criteria:
+- Component renders without errors
+- Visual bars update in real-time during playback
+- Frequency data is correctly calculated from audio samples
+- Click-to-seek works
+- Component cleans up resources properly
+- Visual style matches design requirements
+
+validation:
+- Run: `bun test` for unit tests
+- Manual: Play audio and verify visualization updates
+- Manual: Test seeking and verify visualization follows
+- Performance: Monitor frame rate and CPU usage
+
+notes:
+- Use SolidJS createEffect for reactive updates
+- Keep render loop efficient to maintain 60fps
+- Consider debouncing if processing is too heavy
+- May need to adjust sample rate for performance
+- Visual style should complement existing MergedWaveform design

tasks/real-time-audio-visualization/05-update-player-visualization.md

@@ -0,0 +1,64 @@
+# 05. Update Player component to use realtime visualization
+
+meta:
+  id: real-time-audio-visualization-05
+  feature: real-time-audio-visualization
+  priority: P1
+  depends_on: [real-time-audio-visualization-04]
+  tags: [integration, player]
+
+objective:
+- Replace static waveform display with real-time visualization
+- Update Player.tsx to use RealtimeWaveform component
+- Ensure seamless transition and proper state management
+
+deliverables:
+- Updated src/components/Player.tsx
+- Updated src/components/MergedWaveform.tsx (optional, for fallback)
+- Documentation of changes
+
+steps:
+- Update Player.tsx:
+  - Import RealtimeWaveform component
+  - Replace MergedWaveform with RealtimeWaveform
+  - Pass same props (audioUrl, position, duration, isPlaying, onSeek)
+  - Remove audioUrl from props if no longer needed
+- Test with different audio formats
+- Add fallback handling:
+  - If realtime visualization fails, show static waveform
+  - Graceful degradation for systems without FFTW
+- Update component documentation
+- Test all player controls work with new visualization
+- Verify keyboard shortcuts still work
+- Test seek, pause, resume, volume, speed controls
+
+tests:
+- Unit: Test Player with RealtimeWaveform
+- Integration: Test complete playback flow
+- Integration: Test seek functionality
+- Integration: Test pause/resume
+- Integration: Test volume and speed changes
+- Integration: Test with different audio formats
+- Manual: Verify all player features work correctly
+
+acceptance_criteria:
+- Player displays real-time visualization during playback
+- All player controls work correctly
+- Seek functionality works with visualization
+- Graceful fallback for systems without FFTW
+- No regression in existing functionality
+- Visual style matches design requirements
+
+validation:
+- Run: `bun run build` to verify compilation
+- Run: `bun test` for integration tests
+- Manual: Play various audio files
+- Manual: Test all keyboard shortcuts
+- Performance: Monitor frame rate and CPU usage
+
+notes:
+- Keep MergedWaveform as fallback option
+- Consider showing a loading state while visualizer initializes
+- May need to handle the case where mpv doesn't support audio pipe
+- The visualizer should integrate smoothly with existing Player layout
+- Consider adding a toggle to switch between static and realtime visualization

tasks/real-time-audio-visualization/06-add-visualizer-controls.md

@@ -0,0 +1,78 @@
+# 06. Add visualizer controls and settings
+
+meta:
+  id: real-time-audio-visualization-06
+  feature: real-time-audio-visualization
+  priority: P2
+  depends_on: [real-time-audio-visualization-05]
+  tags: [ui, controls, settings]
+
+objective:
+- Add user controls for visualizer settings
+- Create settings panel for customization
+- Allow users to adjust visualizer parameters
+
+deliverables:
+- src/components/VisualizerSettings.tsx - Settings component
+- Updated src/components/Player.tsx - Settings panel integration
+- src/types/settings.ts - Visualizer settings type definition
+- src/stores/settings.ts - Settings state management
+
+steps:
+- Define visualizer settings types:
+  - Number of bars (resolution)
+  - Sensitivity (autosens toggle + manual value)
+  - Noise reduction level
+  - Frequency cutoffs (low/high)
+  - Bar width and spacing
+  - Color scheme options
+- Create VisualizerSettings component:
+  - Display current settings
+  - Allow adjusting each parameter
+  - Show real-time feedback
+  - Save settings to app store
+- Integrate with Player component:
+  - Add settings button
+  - Show settings panel when toggled
+  - Apply settings to RealtimeWaveform component
+- Update settings state management:
+  - Load saved settings from app store
+  - Save settings on change
+  - Provide default values
+- Create UI for settings:
+  - Keyboard shortcuts for quick adjustment
+  - Visual feedback for changes
+  - Help text for each setting
+- Add settings persistence
+- Create tests for settings component
+
+tests:
+- Unit: Test settings type definitions
+- Unit: Test settings state management
+- Unit: Test VisualizerSettings component
+- Integration: Test settings apply to visualization
+- Integration: Test settings persistence
+- Manual: Test all settings controls
+
+acceptance_criteria:
+- VisualizerSettings component renders correctly
+- All settings can be adjusted
+- Changes apply in real-time
+- Settings persist between sessions
+- Keyboard shortcuts work
+- Component handles invalid settings gracefully
+
+validation:
+- Run: `bun test` for unit tests
+- Run: `bun test` for integration tests
+- Manual: Test all settings
+- Manual: Test keyboard shortcuts
+- Manual: Test settings persistence
+
+notes:
+- Settings should have sensible defaults
+- Some settings may require visualizer re-initialization
+- Consider limiting certain settings to avoid performance issues
+- Add tooltips or help text for complex settings
+- Settings should be optional - users can start without them
+- Keep UI simple and intuitive

tasks/real-time-audio-visualization/README.md

@@ -0,0 +1,30 @@
+# Real-time Audio Visualization
+
+Objective: Integrate cava library for real-time audio visualization in Player component
+
+Status legend: [ ] todo, [~] in-progress, [x] done
+
+Tasks
+- [x] 01 — Copy cavacore library files to project → `01-copy-cavacore-files.md`
+- [ ] 02 — Integrate cavacore library for audio analysis → `02-integrate-cavacore-library.md`
+- [ ] 03 — Create audio stream reader for real-time data → `03-create-audio-stream-reader.md`
+- [ ] 04 — Create realtime waveform component → `04-create-realtime-waveform-component.md`
+- [ ] 05 — Update Player component to use realtime visualization → `05-update-player-visualization.md`
+- [ ] 06 — Add visualizer controls and settings → `06-add-visualizer-controls.md`
+
+Dependencies
+- 01 depends on (none)
+- 02 depends on 01
+- 03 depends on 02
+- 04 depends on 03
+- 05 depends on 04
+- 06 depends on 05
+
+Exit criteria
+- Audio visualization updates in real-time during playback
+- Waveform bars respond to actual audio frequencies
+- Visualizer controls (sensitivity, bar count) work
+- Performance is smooth with 60fps updates
+- All necessary cava files are integrated into project
+
+Note: Files from cava/ directory will be removed after integration

tests/cavacore-smoke.ts

@@ -0,0 +1,58 @@
+/**
+ * Smoke test: load libcavacore.dylib via bun:ffi, init → execute → destroy.
+ * Run: bun tests/cavacore-smoke.ts
+ */
+import { dlopen, FFIType, ptr } from "bun:ffi"
+import { join } from "path"
+
+const libPath = join(import.meta.dir, "..", "src", "native", "libcavacore.dylib")
+
+const lib = dlopen(libPath, {
+  cava_init: {
+    args: [FFIType.i32, FFIType.u32, FFIType.i32, FFIType.i32, FFIType.double, FFIType.i32, FFIType.i32],
+    returns: FFIType.ptr,
+  },
+  cava_execute: {
+    args: [FFIType.ptr, FFIType.i32, FFIType.ptr, FFIType.ptr],
+    returns: FFIType.void,
+  },
+  cava_destroy: {
+    args: [FFIType.ptr],
+    returns: FFIType.void,
+  },
+})
+
+const bars = 10
+const rate = 44100
+const channels = 1
+
+// Init
+const plan = lib.symbols.cava_init(bars, rate, channels, 1, 0.77, 50, 10000)
+if (!plan) {
+  console.error("FAIL: cava_init returned null")
+  process.exit(1)
+}
+console.log("cava_init OK, plan pointer:", plan)
+
+// Generate a 200Hz sine wave test signal
+const bufferSize = 512
+const cavaIn = new Float64Array(bufferSize)
+const cavaOut = new Float64Array(bars * channels)
+
+for (let k = 0; k < 100; k++) {
+  for (let n = 0; n < bufferSize; n++) {
+    cavaIn[n] = Math.sin(2 * Math.PI * 200 / rate * (n + k * bufferSize)) * 20000
+  }
+  lib.symbols.cava_execute(ptr(cavaIn), bufferSize, ptr(cavaOut), plan)
+}
+
+console.log("cava_execute OK, output:", Array.from(cavaOut).map(v => v.toFixed(3)))
+
+// Check that bar 2 (200Hz) has the peak
+const maxIdx = cavaOut.indexOf(Math.max(...cavaOut))
+console.log(`Peak at bar ${maxIdx} (expected ~2 for 200Hz)`)
+
+// Destroy
+lib.symbols.cava_destroy(plan)
+console.log("cava_destroy OK")
+console.log("\nSMOKE TEST PASSED")