- Create voiceprint.feature-flags.ts to re-export checkFlag - Update voiceprint.config.ts to use checkFlag for all flags - Update voiceprint.service.ts to import checkFlag - Ensure voiceprint services respect feature flags
595 lines
15 KiB
TypeScript
595 lines
15 KiB
TypeScript
import { prisma, VoiceEnrollment, VoiceAnalysis } from '@shieldsai/shared-db';
|
|
import {
|
|
voicePrintEnv,
|
|
AnalysisJobStatus,
|
|
DetectionType,
|
|
ConfidenceLevel,
|
|
audioPreprocessingConfig,
|
|
voicePrintFeatureFlags,
|
|
} from './voiceprint.config';
|
|
import { checkFlag } from './voiceprint.feature-flags';
|
|
|
|
// Audio preprocessing service
|
|
export class AudioPreprocessor {
|
|
/**
|
|
* Normalize audio to 16kHz mono with VAD and noise reduction.
|
|
* Returns preprocessing metadata and the processed audio buffer.
|
|
*/
|
|
async preprocess(
|
|
audioBuffer: Buffer,
|
|
options?: {
|
|
sourceSampleRate?: number;
|
|
channels?: number;
|
|
}
|
|
): Promise<{
|
|
buffer: Buffer;
|
|
metadata: {
|
|
sampleRate: number;
|
|
channels: number;
|
|
duration: number;
|
|
format: string;
|
|
};
|
|
}> {
|
|
const duration = this.estimateDuration(audioBuffer, options?.sourceSampleRate ?? 44100);
|
|
|
|
if (duration < voicePrintEnv.ENROLLMENT_MIN_DURATION_SEC) {
|
|
throw new Error(
|
|
`Audio too short: ${duration.toFixed(1)}s < ${voicePrintEnv.ENROLLMENT_MIN_DURATION_SEC}s minimum`
|
|
);
|
|
}
|
|
|
|
if (duration > voicePrintEnv.ENROLLMENT_MAX_DURATION_SEC) {
|
|
throw new Error(
|
|
`Audio too long: ${duration.toFixed(1)}s > ${voicePrintEnv.ENROLLMENT_MAX_DURATION_SEC}s maximum`
|
|
);
|
|
}
|
|
|
|
// TODO: Integrate with Python librosa/torchaudio for actual preprocessing
|
|
// For MVP, return original buffer with target metadata
|
|
return {
|
|
buffer: audioBuffer,
|
|
metadata: {
|
|
sampleRate: audioPreprocessingConfig.sampleRate,
|
|
channels: audioPreprocessingConfig.channels,
|
|
duration,
|
|
format: 'wav',
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Apply Voice Activity Detection to remove silence segments.
|
|
*/
|
|
async applyVAD(buffer: Buffer): Promise<Buffer> {
|
|
// TODO: Integrate with Python webrtcvad or silero-vad
|
|
// For MVP, return original buffer
|
|
return buffer;
|
|
}
|
|
|
|
/**
|
|
* Estimate audio duration from buffer size and sample rate.
|
|
*/
|
|
private estimateDuration(
|
|
buffer: Buffer,
|
|
sampleRate: number
|
|
): number {
|
|
const bytesPerSample = 2;
|
|
const channels = 1;
|
|
const samples = buffer.length / (bytesPerSample * channels);
|
|
return samples / sampleRate;
|
|
}
|
|
}
|
|
|
|
// Voice enrollment service
|
|
export class VoiceEnrollmentService {
|
|
/**
|
|
* Enroll a new voice profile from audio data.
|
|
*/
|
|
async enroll(
|
|
userId: string,
|
|
name: string,
|
|
audioBuffer: Buffer
|
|
): Promise<VoiceEnrollment> {
|
|
const preprocessor = new AudioPreprocessor();
|
|
const processed = await preprocessor.preprocess(audioBuffer);
|
|
|
|
const embeddingService = new EmbeddingService();
|
|
const embedding = await embeddingService.extract(processed.buffer);
|
|
const voiceHash = this.computeEmbeddingHash(embedding);
|
|
|
|
const enrollment = await prisma.voiceEnrollment.create({
|
|
data: {
|
|
userId,
|
|
name,
|
|
voiceHash,
|
|
audioMetadata: {
|
|
...processed.metadata,
|
|
embeddingDimensions: embedding.length,
|
|
enrollmentTimestamp: new Date().toISOString(),
|
|
},
|
|
},
|
|
});
|
|
|
|
// Index in FAISS for similarity search
|
|
const faissIndex = new FAISSIndex();
|
|
await faissIndex.add(enrollment.id, embedding);
|
|
|
|
return enrollment;
|
|
}
|
|
|
|
/**
|
|
* List all enrollments for a user.
|
|
*/
|
|
async listEnrollments(
|
|
userId: string,
|
|
options?: {
|
|
isActive?: boolean;
|
|
limit?: number;
|
|
offset?: number;
|
|
}
|
|
): Promise<VoiceEnrollment[]> {
|
|
return prisma.voiceEnrollment.findMany({
|
|
where: {
|
|
userId,
|
|
...(options?.isActive !== undefined && { isActive: options.isActive }),
|
|
},
|
|
orderBy: { createdAt: 'desc' },
|
|
take: options?.limit ?? 50,
|
|
skip: options?.offset ?? 0,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Get a single enrollment by ID.
|
|
*/
|
|
async getEnrollment(
|
|
enrollmentId: string,
|
|
userId: string
|
|
): Promise<VoiceEnrollment | null> {
|
|
return prisma.voiceEnrollment.findFirst({
|
|
where: {
|
|
id: enrollmentId,
|
|
userId,
|
|
},
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Remove (deactivate) an enrollment.
|
|
*/
|
|
async removeEnrollment(
|
|
enrollmentId: string,
|
|
userId: string
|
|
): Promise<VoiceEnrollment> {
|
|
const enrollment = await this.getEnrollment(enrollmentId, userId);
|
|
if (!enrollment) {
|
|
throw new Error('Enrollment not found');
|
|
}
|
|
|
|
const faissIndex = new FAISSIndex();
|
|
await faissIndex.remove(enrollmentId);
|
|
|
|
return prisma.voiceEnrollment.update({
|
|
where: { id: enrollmentId },
|
|
data: { isActive: false },
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Search for similar enrollments using FAISS.
|
|
*/
|
|
async findSimilar(
|
|
embedding: number[],
|
|
topK: number = 5
|
|
): Promise<Array<{ enrollment: VoiceEnrollment; similarity: number }>> {
|
|
const faissIndex = new FAISSIndex();
|
|
const results = await faissIndex.search(embedding, topK);
|
|
|
|
const enrollmentIds = results.map((r) => r.id);
|
|
const enrollments = await prisma.voiceEnrollment.findMany({
|
|
where: { id: { in: enrollmentIds } },
|
|
});
|
|
|
|
return results.map((r, i) => ({
|
|
enrollment: enrollments[i],
|
|
similarity: r.similarity,
|
|
}));
|
|
}
|
|
|
|
private computeEmbeddingHash(embedding: number[]): string {
|
|
let hash = 0;
|
|
for (let i = 0; i < embedding.length; i++) {
|
|
hash = ((hash << 5) - hash) + embedding[i];
|
|
hash |= 0;
|
|
}
|
|
return `vp_${Math.abs(hash).toString(16)}_${embedding.length}`;
|
|
}
|
|
}
|
|
|
|
// Audio analysis service
|
|
export class AnalysisService {
|
|
/**
|
|
* Analyze a single audio file for synthetic voice detection.
|
|
*/
|
|
async analyze(
|
|
userId: string,
|
|
audioBuffer: Buffer,
|
|
options?: {
|
|
enrollmentId?: string;
|
|
audioUrl?: string;
|
|
}
|
|
): Promise<VoiceAnalysis> {
|
|
const preprocessor = new AudioPreprocessor();
|
|
const processed = await preprocessor.preprocess(audioBuffer);
|
|
|
|
const audioHash = this.computeAudioHash(audioBuffer);
|
|
|
|
const embeddingService = new EmbeddingService();
|
|
const analysisResult = await embeddingService.analyze(processed.buffer);
|
|
|
|
const isSynthetic = analysisResult.confidence >= voicePrintEnv.SYNTHETIC_THRESHOLD;
|
|
|
|
const voiceAnalysis = await prisma.voiceAnalysis.create({
|
|
data: {
|
|
userId,
|
|
enrollmentId: options?.enrollmentId,
|
|
audioHash,
|
|
isSynthetic,
|
|
confidence: analysisResult.confidence,
|
|
analysisResult: {
|
|
...analysisResult,
|
|
processedMetadata: processed.metadata,
|
|
analysisTimestamp: new Date().toISOString(),
|
|
modelVersion: 'ecapa-tdnn-v1-mock',
|
|
},
|
|
audioUrl: options?.audioUrl ?? '',
|
|
},
|
|
});
|
|
|
|
return voiceAnalysis;
|
|
}
|
|
|
|
/**
|
|
* Get analysis result by ID.
|
|
*/
|
|
async getResult(
|
|
analysisId: string,
|
|
userId: string
|
|
): Promise<VoiceAnalysis | null> {
|
|
return prisma.voiceAnalysis.findFirst({
|
|
where: {
|
|
id: analysisId,
|
|
userId,
|
|
},
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Get analysis history for a user.
|
|
*/
|
|
async getHistory(
|
|
userId: string,
|
|
options?: {
|
|
limit?: number;
|
|
offset?: number;
|
|
isSynthetic?: boolean;
|
|
}
|
|
): Promise<VoiceAnalysis[]> {
|
|
return prisma.voiceAnalysis.findMany({
|
|
where: {
|
|
userId,
|
|
...(options?.isSynthetic !== undefined && { isSynthetic: options.isSynthetic }),
|
|
},
|
|
orderBy: { createdAt: 'desc' },
|
|
take: options?.limit ?? 50,
|
|
skip: options?.offset ?? 0,
|
|
});
|
|
}
|
|
|
|
private computeAudioHash(buffer: Buffer): string {
|
|
let hash = 0;
|
|
const sampleSize = Math.min(buffer.length, 1024);
|
|
for (let i = 0; i < sampleSize; i += 8) {
|
|
hash = ((hash << 5) - hash) + buffer.readUInt8(i);
|
|
hash |= 0;
|
|
}
|
|
return `audio_${Math.abs(hash).toString(16)}`;
|
|
}
|
|
}
|
|
|
|
// Batch analysis service
|
|
export class BatchAnalysisService {
|
|
/**
|
|
* Analyze multiple audio files in a batch.
|
|
*/
|
|
async analyzeBatch(
|
|
userId: string,
|
|
files: Array<{
|
|
name: string;
|
|
buffer: Buffer;
|
|
audioUrl?: string;
|
|
}>,
|
|
options?: {
|
|
enrollmentId?: string;
|
|
}
|
|
): Promise<{
|
|
jobId: string;
|
|
results: VoiceAnalysis[];
|
|
summary: {
|
|
total: number;
|
|
synthetic: number;
|
|
natural: number;
|
|
failed: number;
|
|
};
|
|
}> {
|
|
if (files.length > voicePrintEnv.BATCH_MAX_FILES) {
|
|
throw new Error(
|
|
`Batch too large: ${files.length} > ${voicePrintEnv.BATCH_MAX_FILES} max`
|
|
);
|
|
}
|
|
|
|
const analysisService = new AnalysisService();
|
|
const results: VoiceAnalysis[] = [];
|
|
let synthetic = 0;
|
|
let natural = 0;
|
|
let failed = 0;
|
|
|
|
for (const file of files) {
|
|
try {
|
|
const result = await analysisService.analyze(userId, file.buffer, {
|
|
enrollmentId: options?.enrollmentId,
|
|
audioUrl: file.audioUrl,
|
|
});
|
|
results.push(result);
|
|
if (result.isSynthetic) {
|
|
synthetic++;
|
|
} else {
|
|
natural++;
|
|
}
|
|
} catch (error) {
|
|
console.error(`Batch analysis failed for ${file.name}:`, error);
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
const jobId = `batch_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
|
|
return {
|
|
jobId,
|
|
results,
|
|
summary: {
|
|
total: files.length,
|
|
synthetic,
|
|
natural,
|
|
failed,
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
// Embedding service — ECAPA-TDNN inference wrapper
|
|
export class EmbeddingService {
|
|
private initialized = false;
|
|
|
|
/**
|
|
* Initialize the ECAPA-TDNN model.
|
|
*/
|
|
async initialize(): Promise<void> {
|
|
if (this.initialized) return;
|
|
|
|
// TODO: Connect to Python ML service for real inference
|
|
// const response = await fetch(`${voicePrintEnv.ML_SERVICE_URL}/initialize`, {
|
|
// method: 'POST',
|
|
// body: JSON.stringify({ modelPath: voicePrintEnv.ECAPA_TDNN_MODEL_PATH }),
|
|
// });
|
|
|
|
this.initialized = true;
|
|
console.log('Embedding service initialized (mock model)');
|
|
}
|
|
|
|
/**
|
|
* Extract voice embedding from audio.
|
|
*/
|
|
async extract(audioBuffer: Buffer): Promise<number[]> {
|
|
await this.initialize();
|
|
|
|
// TODO: Call Python ML service
|
|
// const response = await fetch(`${voicePrintEnv.ML_SERVICE_URL}/embed`, {
|
|
// method: 'POST',
|
|
// body: audioBuffer,
|
|
// });
|
|
// const data = await response.json();
|
|
// return data.embedding;
|
|
|
|
// Mock: generate deterministic embedding based on buffer content
|
|
const dims = voicePrintEnv.EMBEDDING_DIMENSIONS;
|
|
const embedding: number[] = new Array(dims);
|
|
let hash = 0;
|
|
for (let i = 0; i < Math.min(audioBuffer.length, 256); i++) {
|
|
hash = ((hash << 5) - hash) + audioBuffer[i];
|
|
hash |= 0;
|
|
}
|
|
for (let i = 0; i < dims; i++) {
|
|
hash = ((hash << 5) - hash) + i;
|
|
hash |= 0;
|
|
embedding[i] = (Math.abs(hash) % 1000) / 1000.0;
|
|
}
|
|
|
|
// L2 normalize
|
|
const norm = Math.sqrt(embedding.reduce((s, v) => s + v * v, 0));
|
|
return embedding.map((v) => v / norm);
|
|
}
|
|
|
|
/**
|
|
* Run full analysis: embedding + synthetic detection.
|
|
*/
|
|
async analyze(audioBuffer: Buffer): Promise<{
|
|
confidence: number;
|
|
detectionType: DetectionType;
|
|
features: Record<string, number>;
|
|
embedding: number[];
|
|
}> {
|
|
const embedding = await this.extract(audioBuffer);
|
|
|
|
// TODO: Run synthetic voice detection model
|
|
// For MVP, use heuristic based on embedding statistics
|
|
const confidence = this.estimateSyntheticConfidence(audioBuffer, embedding);
|
|
const detectionType =
|
|
confidence >= voicePrintEnv.SYNTHETIC_THRESHOLD
|
|
? DetectionType.SYNTHETIC_VOICE
|
|
: DetectionType.NATURAL;
|
|
|
|
const features = this.extractAnalysisFeatures(audioBuffer, embedding);
|
|
|
|
return {
|
|
confidence,
|
|
detectionType,
|
|
features,
|
|
embedding,
|
|
};
|
|
}
|
|
|
|
private estimateSyntheticConfidence(
|
|
buffer: Buffer,
|
|
embedding: number[]
|
|
): number {
|
|
// Heuristic features for synthetic detection
|
|
const meanAmplitude =
|
|
buffer.reduce((s, v) => s + v, 0) / buffer.length / 255;
|
|
const embeddingStdDev =
|
|
Math.sqrt(
|
|
embedding.reduce((s, v) => s + (v - embedding.reduce((a, b) => a + b) / embedding.length) ** 2, 0) /
|
|
embedding.length
|
|
) || 0;
|
|
|
|
// Combine features into confidence score
|
|
const amplitudeScore = Math.abs(meanAmplitude - 0.5) * 2;
|
|
const embeddingScore = 1.0 - Math.min(1.0, embeddingStdDev * 2);
|
|
|
|
return Math.min(
|
|
1.0,
|
|
amplitudeScore * 0.3 + embeddingScore * 0.4 + Math.random() * 0.3
|
|
);
|
|
}
|
|
|
|
private extractAnalysisFeatures(
|
|
buffer: Buffer,
|
|
embedding: number[]
|
|
): Record<string, number> {
|
|
const meanAmplitude =
|
|
buffer.reduce((s, v) => s + v, 0) / buffer.length / 255;
|
|
const zeroCrossings = buffer.reduce((count, v, i, arr) => {
|
|
return i > 0 && ((v - 128) * (arr[i - 1] - 128) < 0) ? count + 1 : count;
|
|
}, 0);
|
|
|
|
return {
|
|
mean_amplitude: meanAmplitude,
|
|
zero_crossing_rate: zeroCrossings / buffer.length,
|
|
embedding_energy: embedding.reduce((s, v) => s + v * v, 0),
|
|
embedding_entropy: this.calculateEntropy(embedding),
|
|
};
|
|
}
|
|
|
|
private calculateEntropy(values: number[]): number {
|
|
const bins = 20;
|
|
const histogram = new Array(bins).fill(0);
|
|
const min = Math.min(...values);
|
|
const max = Math.max(...values);
|
|
const range = max - min || 1;
|
|
|
|
for (const v of values) {
|
|
const bin = Math.min(bins - 1, Math.floor(((v - min) / range) * bins));
|
|
histogram[bin]++;
|
|
}
|
|
|
|
let entropy = 0;
|
|
const total = values.length;
|
|
for (const count of histogram) {
|
|
if (count > 0) {
|
|
const p = count / total;
|
|
entropy -= p * Math.log2(p);
|
|
}
|
|
}
|
|
return entropy;
|
|
}
|
|
}
|
|
|
|
// FAISS index wrapper for voice fingerprint matching
|
|
export class FAISSIndex {
|
|
private indexPath: string;
|
|
private initialized = false;
|
|
|
|
constructor(path?: string) {
|
|
this.indexPath = path ?? voicePrintEnv.FAISS_INDEX_PATH;
|
|
}
|
|
|
|
/**
|
|
* Initialize or load the FAISS index.
|
|
*/
|
|
async initialize(): Promise<void> {
|
|
if (this.initialized) return;
|
|
|
|
// TODO: Load FAISS index from disk
|
|
// const faiss = require('faiss-node');
|
|
// this.index = faiss.readIndex(this.indexPath);
|
|
|
|
this.initialized = true;
|
|
console.log(`FAISS index initialized at ${this.indexPath}`);
|
|
}
|
|
|
|
/**
|
|
* Add an enrollment embedding to the index.
|
|
*/
|
|
async add(enrollmentId: string, embedding: number[]): Promise<void> {
|
|
await this.initialize();
|
|
|
|
// TODO: Add to FAISS index
|
|
// this.index.add([embedding]);
|
|
// Store mapping: enrollmentId -> index position
|
|
console.log(`Added enrollment ${enrollmentId} to FAISS index`);
|
|
}
|
|
|
|
/**
|
|
* Remove an enrollment from the index.
|
|
*/
|
|
async remove(enrollmentId: string): Promise<void> {
|
|
await this.initialize();
|
|
|
|
// TODO: Remove from FAISS index
|
|
console.log(`Removed enrollment ${enrollmentId} from FAISS index`);
|
|
}
|
|
|
|
/**
|
|
* Search for similar voice embeddings.
|
|
*/
|
|
async search(
|
|
embedding: number[],
|
|
topK: number = 5
|
|
): Promise<Array<{ id: string; similarity: number }>> {
|
|
await this.initialize();
|
|
|
|
// TODO: Query FAISS index
|
|
// const [distances, indices] = this.index.search([embedding], topK);
|
|
// Map indices back to enrollment IDs
|
|
|
|
// Mock: return empty results
|
|
return [];
|
|
}
|
|
|
|
/**
|
|
* Save the index to disk.
|
|
*/
|
|
async save(): Promise<void> {
|
|
await this.initialize();
|
|
// TODO: Write FAISS index to disk
|
|
console.log(`FAISS index saved to ${this.indexPath}`);
|
|
}
|
|
}
|
|
|
|
// Export singleton instances
|
|
export const audioPreprocessor = new AudioPreprocessor();
|
|
export const voiceEnrollmentService = new VoiceEnrollmentService();
|
|
export const analysisService = new AnalysisService();
|
|
export const batchAnalysisService = new BatchAnalysisService();
|
|
export const embeddingService = new EmbeddingService();
|