for first push

This commit is contained in:
2026-04-29 16:29:03 -04:00
parent 218de3b03b
commit 509259bcf2
19 changed files with 1911 additions and 2 deletions

View File

@@ -16,6 +16,7 @@
"@shieldai/db": "0.1.0",
"@shieldai/types": "0.1.0",
"fastify": "^5.2.0",
"@shieldai/darkwatch": "0.1.0"
"@shieldai/darkwatch": "0.1.0",
"@shieldai/voiceprint": "0.1.0"
}
}

View File

@@ -13,3 +13,10 @@ export function darkwatchRoutes(fastify: FastifyInstance) {
root.register(scans, { prefix: "/scan" });
}, { prefix: "/api/v1/darkwatch" });
}
export function voiceprintRoutes(fastify: FastifyInstance) {
fastify.register(async (root) => {
const voiceprint = (await import("./voiceprint.routes")).voiceprintRoutes;
root.register(voiceprint);
}, { prefix: "/api/v1/voiceprint" });
}

View File

@@ -0,0 +1,94 @@
import { FastifyInstance } from "fastify";
import { VoiceEnrollmentService } from "@shieldai/voiceprint";
import { AnalysisService } from "@shieldai/voiceprint";
import { BatchAnalysisService } from "@shieldai/voiceprint";
export function voiceprintRoutes(fastify: FastifyInstance) {
const enrollmentService = new VoiceEnrollmentService();
const analysisService = new AnalysisService();
const batchService = new BatchAnalysisService();
fastify.post("/enroll", async (request, reply) => {
const userId = (request.user as { id: string })?.id;
if (!userId) return reply.code(401).send({ error: "User not authenticated" });
const body = request.body as { label: string; audio: string; sampleRate?: number };
const audioBuffer = Buffer.from(body.audio, "base64");
const enrollment = await enrollmentService.enroll(
{ label: body.label, audioBuffer, sampleRate: body.sampleRate },
userId
);
return reply.code(201).send(enrollment);
});
fastify.get("/enrollments", async (request, reply) => {
const userId = (request.user as { id: string })?.id;
if (!userId) return reply.code(401).send({ error: "User not authenticated" });
const enrollments = await enrollmentService.listEnrollments(userId);
return reply.send(enrollments);
});
fastify.delete("/enrollments/:id", async (request, reply) => {
const userId = (request.user as { id: string })?.id;
if (!userId) return reply.code(401).send({ error: "User not authenticated" });
const enrollmentId = (request.params as { id: string }).id;
const result = await enrollmentService.removeEnrollment(userId, enrollmentId);
return reply.send({ removed: result });
});
fastify.post("/analyze", async (request, reply) => {
const userId = (request.user as { id: string })?.id;
if (!userId) return reply.code(401).send({ error: "User not authenticated" });
const body = request.body as { audio: string; sampleRate?: number; analysisType?: string };
const audioBuffer = Buffer.from(body.audio, "base64");
const result = await analysisService.analyze(
{ audioBuffer, sampleRate: body.sampleRate, analysisType: body.analysisType },
userId
);
return reply.code(201).send(result);
});
fastify.get("/results/:id", async (request, reply) => {
const jobId = (request.params as { id: string }).id;
const result = await analysisService.getResult(jobId);
if (!result) return reply.code(404).send({ error: "Analysis result not found" });
return reply.send(result);
});
fastify.get("/results", async (request, reply) => {
const userId = (request.user as { id: string })?.id;
if (!userId) return reply.code(401).send({ error: "User not authenticated" });
const limit = parseInt((request.query as { limit?: string }).limit || "20", 10);
const results = await analysisService.getUserResults(userId, limit);
return reply.send(results);
});
fastify.post("/batch", async (request, reply) => {
const userId = (request.user as { id: string })?.id;
if (!userId) return reply.code(401).send({ error: "User not authenticated" });
const body = request.body as {
files: Array<{ name: string; audio: string; sampleRate?: number }>;
analysisType?: string;
};
const audioBuffers = body.files.map((f) => ({
name: f.name,
buffer: Buffer.from(f.audio, "base64"),
sampleRate: f.sampleRate,
}));
const result = await batchService.analyzeBatch(
{ audioBuffers, analysisType: body.analysisType },
userId
);
return reply.code(201).send(result);
});
}

View File

@@ -2,7 +2,7 @@ import Fastify from "fastify";
import cors from "@fastify/cors";
import helmet from "@fastify/helmet";
import sensible from "@fastify/sensible";
import { darkwatchRoutes } from "./routes";
import { darkwatchRoutes, voiceprintRoutes } from "./routes";
const app = Fastify({
logger: {
@@ -16,6 +16,7 @@ async function bootstrap() {
await app.register(sensible);
await app.register(darkwatchRoutes);
await app.register(voiceprintRoutes);
app.get("/health", async () => ({ status: "ok", timestamp: new Date().toISOString() }));

View File

@@ -0,0 +1,316 @@
/**
* Audio Processing Pipeline for Real-Time Analysis
* Coordinates WebRTC stream capture with VoicePreprocess for continuous analysis
*/
import { WebRTCStreamCapture, createWebRTCCapture } from './stream-capture';
// Type definitions for real-time processing
export interface AudioChunk {
id: string;
timestamp: number;
data: Float32Array;
duration: number;
}
export interface VoiceprintResult {
chunkId: string;
timestamp: number;
features: AudioFeatures;
embedding: number[];
confidence: number;
status: 'complete' | 'error';
}
// Audio chunk configuration
export interface AudioChunkConfig {
chunkDuration: number;
overlapDuration: number;
sampleRate: number;
}
// Preprocessor interfaces (copied from AudioPreprocessor for standalone usage)
export interface PreprocessedAudio {
audio: Buffer;
sampleRate: number;
channels: number;
durationSec: number;
}
export interface AudioFeatures {
mfccs: number[][];
zeroCrossingRate: number;
spectralCentroid: number;
spectralRollOff: number;
durationSec: number;
}
interface PipelineConfig {
chunkDuration: number; // 5000ms
overlapDuration: number; // 1000ms
sampleRate: number; // 16000 Hz
onAnalysisComplete?: (result: VoiceprintResult) => void;
onStreamError?: (error: Error) => void;
onError?: (error: Error) => void;
}
const DEFAULT_CONFIG: PipelineConfig = {
chunkDuration: 5000,
overlapDuration: 1000,
sampleRate: 16000
};
export class AudioPipeline {
private streamCapture: WebRTCStreamCapture | null = null;
private isRunning: boolean = false;
private chunkBuffer: AudioChunk[] = [];
private maxBufferLength: number = 10;
private onChunkReady?: (chunk: AudioChunk) => void;
private onAnalysisComplete?: (result: VoiceprintResult) => void;
private onPipelineError?: (error: Error) => void;
constructor(private config: PipelineConfig = DEFAULT_CONFIG) {}
/**
* Initialize pipeline components
*/
async initialize(): Promise<void> {
// Initialize WebRTC stream capture
this.streamCapture = createWebRTCCapture({
chunkDuration: this.config.chunkDuration,
overlapDuration: this.config.overlapDuration,
sampleRate: this.config.sampleRate
});
// Connect WebRTC chunk processing
this.streamCapture.onChunkReady = (rawAudio, timestamp) => {
this.handleRawChunk(rawAudio, timestamp);
};
// Connect stream error handling
this.streamCapture.onStreamError = (error) => {
this.onPipelineError?.(error);
};
console.log('[Pipeline] Initialized');
}
/**
* Process raw audio chunk from WebRTC
*/
private async handleRawChunk(rawAudio: Float32Array, timestamp: number): Promise<void> {
try {
// Create audio chunk
const chunk: AudioChunk = {
id: `chunk-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
timestamp,
data: rawAudio,
duration: this.config.chunkDuration / 1000
};
// Add to buffer
this.chunkBuffer.push(chunk);
// Maintain buffer size
if (this.chunkBuffer.length > this.maxBufferLength) {
const removed = this.chunkBuffer.shift();
if (removed) {
// Process removed chunk with overlap
await this.processChunk(removed);
}
}
// Process current chunk
await this.processChunk(chunk);
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
console.error('[Pipeline] Error processing chunk:', err);
this.onPipelineError?.(err);
}
}
/**
* Convert Float32Array to 16-bit PCM Buffer
*/
private float32ArrayToBuffer(floatData: Float32Array): Buffer {
const int16Array = new Int16Array(floatData.length);
for (let i = 0; i < floatData.length; i++) {
// Clamp and scale to 16-bit range
const clamped = Math.max(-1, Math.min(1, floatData[i]));
int16Array[i] = Math.round(clamped * 32767);
}
return Buffer.from(int16Array.buffer);
}
/**
* Process a single audio chunk (mock implementation)
*/
private async processChunk(chunk: AudioChunk): Promise<VoiceprintResult> {
try {
// Generate mock features
// Convert Float32Array to Buffer properly
const int16Array = new Int16Array(chunk.data.length);
for (let i = 0; i < chunk.data.length; i++) {
const clamped = Math.max(-1, Math.min(1, chunk.data[i]));
int16Array[i] = Math.round(clamped * 32767);
}
const dataBuffer = Buffer.from(int16Array.buffer);
const features: AudioFeatures = await extractMockFeatures(dataBuffer, chunk.duration);
// Generate embedding (placeholder - would use actual embedding service)
const embedding = this.generatePlaceholderEmbedding(features);
// Create result
const result: VoiceprintResult = {
chunkId: chunk.id,
timestamp: chunk.timestamp,
features,
embedding,
confidence: 0.95, // Placeholder - would come from actual analysis
status: 'complete'
};
// Emit result
this.onAnalysisComplete?.(result);
return result;
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
console.error('[Pipeline] Preprocessing error:', err);
this.onPipelineError?.(err);
throw err;
}
}
/**
* Generate placeholder embedding (would use actual embedding service)
*/
private generatePlaceholderEmbedding(features: AudioFeatures): number[] {
// Placeholder embedding - would be replaced with actual embedding generation
const embedding: number[] = [];
// Use spectral features as proxy for embedding
embedding.push(features.spectralCentroid);
embedding.push(features.spectralRollOff);
embedding.push(features.zeroCrossingRate);
// MFCC summary (first few coefficients)
if (features.mfccs && features.mfccs.length > 0) {
for (let i = 0; i < Math.min(5, features.mfccs[0].length); i++) {
embedding.push(features.mfccs[0][i]);
}
}
// Normalize and pad
while (embedding.length < 128) {
embedding.push(0);
}
return embedding;
}
/**
* Start the real-time analysis pipeline
*/
async start(): Promise<void> {
if (this.isRunning) {
console.log('[Pipeline] Already running');
return;
}
try {
await this.initialize();
if (this.streamCapture) {
await this.streamCapture.start();
}
this.isRunning = true;
console.log('[Pipeline] Real-time analysis started');
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
console.error('[Pipeline] Failed to start:', err);
this.onPipelineError?.(err);
throw err;
}
}
/**
* Stop the pipeline
*/
async stop(): Promise<void> {
if (!this.isRunning) {
return;
}
try {
// Drain remaining buffer
while (this.chunkBuffer.length > 0) {
const chunk = this.chunkBuffer.shift();
if (chunk) {
await this.processChunk(chunk);
}
}
// Stop WebRTC capture
if (this.streamCapture) {
this.streamCapture.stop();
}
this.isRunning = false;
console.log('[Pipeline] Stopped');
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
console.error('[Pipeline] Error stopping:', err);
this.onPipelineError?.(err);
throw err;
}
}
/**
* Get pipeline status
*/
getStatus(): {
isRunning: boolean;
bufferLength: number;
streamActive: boolean;
} {
return {
isRunning: this.isRunning,
bufferLength: this.chunkBuffer.length,
streamActive: this.streamCapture?.isRecording || false
};
}
}
/**
* Extract mock features from audio buffer
*/
async function extractMockFeatures(buffer: Buffer, duration: number): Promise<AudioFeatures> {
// Simple mock feature extraction for demonstration
// In production, this would use actual audio processing
const numMfccs = 13;
const mfccs: number[][] = [];
// Generate mock MFCCs based on buffer hash
const bufferHash = buffer.reduce((acc, byte) => acc + byte, 0);
for (let i = 0; i < numMfccs; i++) {
const coefficients: number[] = [];
for (let j = 0; j < 20; j++) {
coefficients.push(Math.abs(Math.sin((i * j + bufferHash) * 0.1)) * 0.5 + 0.25);
}
mfccs.push(coefficients);
}
return {
mfccs,
zeroCrossingRate: 0.02 + Math.random() * 0.03,
spectralCentroid: 1000 + Math.random() * 2000,
spectralRollOff: 3000 + Math.random() * 1000,
durationSec: duration
};
}

View File

@@ -0,0 +1,184 @@
/**
* WebRTC Audio Stream Capture
* Captures audio from screen/audio sharing using WebRTC APIs
* Implements 5-second chunks with 1-second overlap for sliding window analysis
*/
interface WebRTCStreamConfig {
chunkDuration: number; // 5000ms
overlapDuration: number; // 1000ms
sampleRate: number; // 16000 Hz for voiceprint compatibility
}
const DEFAULT_CONFIG: WebRTCStreamConfig = {
chunkDuration: 5000,
overlapDuration: 1000,
sampleRate: 16000
};
export class WebRTCStreamCapture {
private stream: MediaStream | null = null;
private audioContext: AudioContext | null = null;
private analyser: AnalyserNode | null = null;
private source: MediaStreamAudioSourceNode | null = null;
private _isRecording: boolean = false;
private buffer: Float32Array = new Float32Array(0);
public onChunkReady?: (chunk: Float32Array, timestamp: number) => void;
public onStreamError?: (error: Error) => void;
constructor(private config: WebRTCStreamConfig = DEFAULT_CONFIG) {}
/**
* Check if currently recording
*/
public isRecording: boolean = false;
/**
* Start capturing audio from screen/audio sharing
*/
async start(): Promise<void> {
if (this.isRecording) {
console.log('[WebRTC] Already recording');
return;
}
try {
// Request screen/audio capture with audio
this.stream = await navigator.mediaDevices.getDisplayMedia({
video: true, // Required for audio capture
audio: true
});
// Stop any existing tracks
this.stream.getTracks().forEach(track => track.stop());
// Create audio context and analyser
this.audioContext = new AudioContext({
sampleRate: this.config.sampleRate
});
this.analyser = this.audioContext.createAnalyser();
this.analyser.fftSize = 2048;
// Connect stream to audio graph
this.source = this.audioContext.createMediaStreamSource(this.stream);
this.source.connect(this.analyser);
this.isRecording = true;
console.log('[WebRTC] Stream capture started');
// Start processing loop
this.processAudio();
// Handle stream termination
this.stream.getVideoTracks()[0].onended = () => {
console.log('[WebRTC] User stopped sharing');
this.stop();
};
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
console.error('[WebRTC] Failed to start stream capture:', err);
this.onStreamError?.(err);
throw err;
}
}
/**
* Process audio in real-time with sliding window
*/
private processAudio(): void {
if (!this.audioContext || !this.analyser || !this.isRecording) return;
if (!this.analyser) return;
const bufferLength = this.analyser.fftSize;
const buffer = new Float32Array(bufferLength);
const processFrame = () => {
if (!this.isRecording) return;
if (!this.analyser) return;
this.analyser.getFloatTimeDomainData(buffer);
// Get current timestamp
const timestamp = this.audioContext?.currentTime ?? 0;
// Extract audio data for current frame
// Use first 512 samples for voice analysis (reduced for faster processing)
const audioData = buffer.slice(0, 512);
// Prepare chunk for analysis
if (audioData.length > 0) {
this.onChunkReady?.(audioData, timestamp);
}
// Schedule next frame with overlap
const frameDuration = this.config.chunkDuration - this.config.overlapDuration;
setTimeout(processFrame, frameDuration);
};
processFrame();
}
/**
* Stop audio capture
*/
stop(): void {
this._isRecording = false;
if (this.stream) {
this.stream.getTracks().forEach(track => track.stop());
this.stream = null;
}
if (this.source) {
this.source.disconnect();
this.source = null;
}
if (this.analyser) {
this.analyser.disconnect();
this.analyser = null;
}
if (this.audioContext) {
this.audioContext.close();
this.audioContext = null;
}
console.log('[WebRTC] Stream capture stopped');
}
/**
* Get stream metadata
*/
getMetadata(): {
isActive: boolean;
sampleRate: number;
channels: number;
} {
if (!this.stream) {
return { isActive: false, sampleRate: 0, channels: 0 };
}
const audioTrack = this.stream.getAudioTracks()[0];
if (!audioTrack) {
return { isActive: true, sampleRate: this.config.sampleRate, channels: 1 };
}
return {
isActive: true,
sampleRate: this.config.sampleRate,
channels: audioTrack.getSettings().channelCount || 1
};
}
}
/**
* Factory function for creating stream capture with auto-start
*/
export function createWebRTCCapture(config?: WebRTCStreamConfig): WebRTCStreamCapture {
return new WebRTCStreamCapture(config || DEFAULT_CONFIG);
}

View File

@@ -0,0 +1,22 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"lib": ["ES2020", "DOM"],
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"declaration": true,
"resolveJsonModule": true,
"moduleResolution": "node",
"baseUrl": ".",
"paths": {
"@/*": ["src/*"]
}
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}

View File

@@ -57,6 +57,25 @@ enum DataSource {
HONEYPOT
}
enum AnalysisJobStatus {
PENDING
RUNNING
COMPLETED
FAILED
}
enum AnalysisType {
SYNTHETIC_DETECTION
VOICE_MATCH
BATCH
}
enum DetectionVerdict {
NATURAL
SYNTHETIC
UNCERTAIN
}
model User {
id String @id @default(uuid())
email String @unique
@@ -138,3 +157,54 @@ model ScanJob {
@@index([userId, status])
@@index([createdAt])
}
model VoiceEnrollment {
id String @id @default(uuid())
userId String
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
label String
embeddingVector Float[]
embeddingDim Int @default(192)
audioFilePath String?
sampleRate Int @default(16000)
durationSec Float?
createdAt DateTime @default(now())
updatedAt DateTime @updatedAt
@@index([userId])
@@index([embeddingDim])
}
model AnalysisJob {
id String @id @default(uuid())
userId String
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
analysisType AnalysisType
audioFilePath String
status AnalysisJobStatus @default(PENDING)
result AnalysisResult?
errorMessage String?
completedAt DateTime?
createdAt DateTime @default(now())
@@index([userId, status])
@@index([createdAt])
}
model AnalysisResult {
id String @id @default(uuid())
analysisJobId String @unique
analysisJob AnalysisJob @relation(fields: [analysisJobId], references: [id], onDelete: Cascade)
syntheticScore Float
verdict DetectionVerdict
matchedEnrollmentId String?
matchedSimilarity Float?
confidence Float
processingTimeMs Int
modelVersion String?
metadata String?
createdAt DateTime @default(now())
@@index([analysisJobId])
@@index([verdict])
}

View File

@@ -0,0 +1,54 @@
import { Queue, Worker } from "bullmq";
import { Redis } from "ioredis";
import { AnalysisService } from "@shieldai/voiceprint";
const redisUrl = process.env.REDIS_URL || "redis://localhost:6379";
const connection = new Redis(redisUrl);
const analysisQueue = new Queue("voiceprint-analysis", { connection });
const analysisWorker = new Worker(
"voiceprint-analysis",
async (job) => {
const { userId, audioBuffer, sampleRate, analysisType } = job.data;
const analysisService = new AnalysisService();
const result = await analysisService.analyze(
{
audioBuffer: Buffer.from(audioBuffer, "base64"),
sampleRate,
analysisType,
},
userId
);
return { jobId: result.jobId, completedAt: new Date().toISOString() };
},
{ connection, concurrency: 2 }
);
analysisWorker.on("completed", (job) => {
console.log(`[VoicePrint] Job ${job.id} completed: ${JSON.stringify(job.returnvalue)}`);
});
analysisWorker.on("failed", (job, err) => {
console.error(`[VoicePrint] Job ${job.id} failed: ${err.message}`);
});
export async function addAnalysisJob(
userId: string,
audioBuffer: Buffer,
sampleRate?: number,
analysisType?: string
) {
return analysisQueue.add("analyze", {
userId,
audioBuffer: audioBuffer.toString("base64"),
sampleRate,
analysisType,
}, {
attempts: 3,
backoff: { type: "exponential", delay: 5000 },
jobId: `vp-${userId}-${Date.now()}`,
});
}
console.log("[VoicePrint] Analysis worker started");

View File

@@ -82,3 +82,62 @@ export interface AlertInput {
channel: AlertChannel;
dedupKey: string;
}
export const AnalysisJobStatus = {
PENDING: "PENDING",
RUNNING: "RUNNING",
COMPLETED: "COMPLETED",
FAILED: "FAILED",
} as const;
export type AnalysisJobStatus = (typeof AnalysisJobStatus)[keyof typeof AnalysisJobStatus];
export const AnalysisType = {
SYNTHETIC_DETECTION: "SYNTHETIC_DETECTION",
VOICE_MATCH: "VOICE_MATCH",
BATCH: "BATCH",
} as const;
export type AnalysisType = (typeof AnalysisType)[keyof typeof AnalysisType];
export const DetectionVerdict = {
NATURAL: "NATURAL",
SYNTHETIC: "SYNTHETIC",
UNCERTAIN: "UNCERTAIN",
} as const;
export type DetectionVerdict = (typeof DetectionVerdict)[keyof typeof DetectionVerdict];
export interface VoiceEnrollmentInput {
label: string;
audioBuffer: Buffer;
sampleRate?: number;
}
export interface AnalyzeAudioInput {
audioBuffer: Buffer;
sampleRate?: number;
analysisType?: AnalysisType;
}
export interface BatchAnalyzeInput {
audioBuffers: Array<{ name: string; buffer: Buffer; sampleRate?: number }>;
analysisType?: AnalysisType;
}
export interface AnalysisResultOutput {
jobId: string;
syntheticScore: number;
verdict: DetectionVerdict;
confidence: number;
matchedEnrollmentId?: string;
matchedSimilarity?: number;
processingTimeMs: number;
modelVersion?: string;
}
export interface VoiceEnrollmentOutput {
id: string;
label: string;
embeddingDim: number;
sampleRate: number;
durationSec?: number;
createdAt: Date;
}