Compare commits
8 Commits
35e9f7e812
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 6c4d0b91ca | |||
| 0c9b14a54b | |||
| 56016a6124 | |||
| 01ffe79bbe | |||
| 0f997b639f | |||
| 726aafef74 | |||
| 31e0b39794 | |||
| a653c77959 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -4,3 +4,4 @@ dist
|
|||||||
*.log
|
*.log
|
||||||
.DS_Store
|
.DS_Store
|
||||||
load-tests/voiceprint/results/
|
load-tests/voiceprint/results/
|
||||||
|
.turbo
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
# ShieldAI Rollback Runbook
|
# ShieldAI Rollback Runbook
|
||||||
|
|
||||||
> **Last updated:** 2026-05-09
|
> **Last updated:** 2026-05-12
|
||||||
> **Owner:** Senior Engineer
|
> **Owner:** Senior Engineer
|
||||||
> **Parent:** [FRE-4574](/FRE/issues/FRE-4574) ShieldAI Production Infrastructure & CI/CD Pipeline
|
> **Parent:** [FRE-4574](/FRE/issues/FRE-4574) ShieldAI Production Infrastructure & CI/CD Pipeline
|
||||||
|
> **Reviewed by:** Code Reviewer (FRE-4808) on 2026-05-12
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -268,30 +268,25 @@ export function mixedWorkload() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Individual endpoint scenarios — each makes exactly 1 HTTP call per iteration
|
// Individual endpoint scenarios — each makes exactly 1 HTTP call per iteration
|
||||||
|
// NOTE: constant-arrival-rate executor does not pass setup() data to scenario functions.
|
||||||
|
// Standalone runs always use fake tokens (expected 401/403). For real-token testing,
|
||||||
|
// run as part of the mixedWorkload scenario or switch to vus executor.
|
||||||
export function loginOnly() {
|
export function loginOnly() {
|
||||||
testLogin();
|
testLogin();
|
||||||
sleep(0.1);
|
sleep(0.1);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function logoutOnly(data) {
|
export function logoutOnly() {
|
||||||
if (data && data.warmupSuccess) {
|
const poolEntry = tokenPool[Math.floor(Math.random() * tokenPool.length)];
|
||||||
testLogout(data.accessToken, data.refreshToken);
|
console.warn('[logoutOnly] Using fake token (constant-arrival-rate does not pass setup() data)');
|
||||||
} else {
|
testLogout(poolEntry.accessToken, poolEntry.refreshToken);
|
||||||
const poolEntry = tokenPool[Math.floor(Math.random() * tokenPool.length)];
|
|
||||||
console.warn('[logoutOnly] Using fake token (warmup skipped or failed)');
|
|
||||||
testLogout(poolEntry.accessToken, poolEntry.refreshToken);
|
|
||||||
}
|
|
||||||
sleep(0.1);
|
sleep(0.1);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function refreshOnly(data) {
|
export function refreshOnly() {
|
||||||
if (data && data.warmupSuccess) {
|
const poolEntry = tokenPool[Math.floor(Math.random() * tokenPool.length)];
|
||||||
testRefresh(data.refreshToken);
|
console.warn('[refreshOnly] Using fake token (constant-arrival-rate does not pass setup() data)');
|
||||||
} else {
|
testRefresh(poolEntry.refreshToken);
|
||||||
const poolEntry = tokenPool[Math.floor(Math.random() * tokenPool.length)];
|
|
||||||
console.warn('[refreshOnly] Using fake token (warmup skipped or failed)');
|
|
||||||
testRefresh(poolEntry.refreshToken);
|
|
||||||
}
|
|
||||||
sleep(0.1);
|
sleep(0.1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -28,26 +28,27 @@ echo "Duration: ${DURATION:-300s}"
|
|||||||
echo "Base URL: ${DARKWATCH_BASE_URL:-http://localhost:3000}"
|
echo "Base URL: ${DARKWATCH_BASE_URL:-http://localhost:3000}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
EXIT_CODE=0
|
||||||
case "$SCENARIO" in
|
case "$SCENARIO" in
|
||||||
mixed)
|
mixed)
|
||||||
k6 run darkwatch-auth.js \
|
k6 run darkwatch-auth.js \
|
||||||
--summary-export "$OUTPUT_DIR/summary-${TIMESTAMP}.json" \
|
--summary-export "$OUTPUT_DIR/summary-${TIMESTAMP}.json" \
|
||||||
--out json="$OUTPUT_DIR/results-${TIMESTAMP}.json"
|
--out json="$OUTPUT_DIR/results-${TIMESTAMP}.json" || EXIT_CODE=$?
|
||||||
;;
|
;;
|
||||||
login)
|
login)
|
||||||
k6 run --scenario login_only darkwatch-auth.js \
|
k6 run --scenario login_only darkwatch-auth.js \
|
||||||
--summary-export "$OUTPUT_DIR/summary-${TIMESTAMP}.json" \
|
--summary-export "$OUTPUT_DIR/summary-${TIMESTAMP}.json" \
|
||||||
--out json="$OUTPUT_DIR/results-${TIMESTAMP}.json"
|
--out json="$OUTPUT_DIR/results-${TIMESTAMP}.json" || EXIT_CODE=$?
|
||||||
;;
|
;;
|
||||||
logout)
|
logout)
|
||||||
k6 run --scenario logout_only darkwatch-auth.js \
|
k6 run --scenario logout_only darkwatch-auth.js \
|
||||||
--summary-export "$OUTPUT_DIR/summary-${TIMESTAMP}.json" \
|
--summary-export "$OUTPUT_DIR/summary-${TIMESTAMP}.json" \
|
||||||
--out json="$OUTPUT_DIR/results-${TIMESTAMP}.json"
|
--out json="$OUTPUT_DIR/results-${TIMESTAMP}.json" || EXIT_CODE=$?
|
||||||
;;
|
;;
|
||||||
refresh)
|
refresh)
|
||||||
k6 run --scenario refresh_only darkwatch-auth.js \
|
k6 run --scenario refresh_only darkwatch-auth.js \
|
||||||
--summary-export "$OUTPUT_DIR/summary-${TIMESTAMP}.json" \
|
--summary-export "$OUTPUT_DIR/summary-${TIMESTAMP}.json" \
|
||||||
--out json="$OUTPUT_DIR/results-${TIMESTAMP}.json"
|
--out json="$OUTPUT_DIR/results-${TIMESTAMP}.json" || EXIT_CODE=$?
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Unknown scenario: $SCENARIO"
|
echo "Unknown scenario: $SCENARIO"
|
||||||
@@ -56,8 +57,6 @@ case "$SCENARIO" in
|
|||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
EXIT_CODE=$?
|
|
||||||
|
|
||||||
if [[ $EXIT_CODE -eq 0 ]]; then
|
if [[ $EXIT_CODE -eq 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "✅ All thresholds passed!"
|
echo "✅ All thresholds passed!"
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
// dd-trace must be initialized before any other module is loaded for auto-instrumentation
|
||||||
|
import '@shieldai/monitoring/datadog-init';
|
||||||
import Fastify from 'fastify';
|
import Fastify from 'fastify';
|
||||||
import cors from '@fastify/cors';
|
import cors from '@fastify/cors';
|
||||||
import helmet from '@fastify/helmet';
|
import helmet from '@fastify/helmet';
|
||||||
@@ -8,7 +10,6 @@ import { errorHandlingMiddleware } from './middleware/error-handling.middleware'
|
|||||||
import { loggingMiddleware } from './middleware/logging.middleware';
|
import { loggingMiddleware } from './middleware/logging.middleware';
|
||||||
import { apiEnv, loggingConfig, getCorsOrigins } from './config/api.config';
|
import { apiEnv, loggingConfig, getCorsOrigins } from './config/api.config';
|
||||||
import { routes } from './routes';
|
import { routes } from './routes';
|
||||||
import { initDatadog, initSentry } from '@shieldai/monitoring';
|
|
||||||
|
|
||||||
const fastify = Fastify({
|
const fastify = Fastify({
|
||||||
logger: loggingConfig,
|
logger: loggingConfig,
|
||||||
@@ -16,10 +17,6 @@ const fastify = Fastify({
|
|||||||
maxParamLength: 500,
|
maxParamLength: 500,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Initialize monitoring (must be first import for auto-instrumentation)
|
|
||||||
initDatadog();
|
|
||||||
initSentry();
|
|
||||||
|
|
||||||
// Register plugins
|
// Register plugins
|
||||||
async function registerPlugins() {
|
async function registerPlugins() {
|
||||||
// CORS configuration
|
// CORS configuration
|
||||||
|
|||||||
@@ -46,9 +46,10 @@ export async function authMiddleware(fastify: FastifyInstance) {
|
|||||||
if (apiKey) {
|
if (apiKey) {
|
||||||
// In production, validate API key against database
|
// In production, validate API key against database
|
||||||
authReq.apiKey = apiKey;
|
authReq.apiKey = apiKey;
|
||||||
|
const apiKeyPrefix = apiKey.slice(0, 8);
|
||||||
authReq.user = {
|
authReq.user = {
|
||||||
id: `api-${apiKey}`,
|
id: `api-${apiKeyPrefix}...`,
|
||||||
email: `api-${apiKey}@services.internal`,
|
email: `api-${apiKeyPrefix}@services.internal`,
|
||||||
role: 'service',
|
role: 'service',
|
||||||
};
|
};
|
||||||
authReq.authType = 'api-key';
|
authReq.authType = 'api-key';
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
|
import { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
|
||||||
import { emitLatency, emitRequestCount, emitError } from '@shieldai/monitoring';
|
import { emitBatchMetrics, emitError } from '@shieldai/monitoring';
|
||||||
|
|
||||||
const SERVICE_NAME = process.env.DD_SERVICE || 'shieldai-api';
|
const SERVICE_NAME = process.env.DD_SERVICE || 'shieldai-api';
|
||||||
|
|
||||||
@@ -10,15 +10,38 @@ export async function monitoringMiddleware(fastify: FastifyInstance) {
|
|||||||
const method = request.method;
|
const method = request.method;
|
||||||
const url = request.url;
|
const url = request.url;
|
||||||
|
|
||||||
// Emit request count
|
// Batch all metrics into a single PutMetricDataCommand to avoid rate limits
|
||||||
await emitRequestCount(SERVICE_NAME, statusCode);
|
await emitBatchMetrics({
|
||||||
|
serviceName: SERVICE_NAME,
|
||||||
|
data: [
|
||||||
|
{
|
||||||
|
metricName: 'api_requests',
|
||||||
|
value: 1,
|
||||||
|
unit: 'Count',
|
||||||
|
dimensions: { status_class: String(Math.floor(statusCode / 100)) + 'xx' },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
metricName: 'api_latency',
|
||||||
|
value: responseTime,
|
||||||
|
unit: 'Milliseconds',
|
||||||
|
dimensions: { percentile: 'p50' },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
metricName: 'api_latency',
|
||||||
|
value: responseTime,
|
||||||
|
unit: 'Milliseconds',
|
||||||
|
dimensions: { percentile: 'p95' },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
metricName: 'api_latency',
|
||||||
|
value: responseTime,
|
||||||
|
unit: 'Milliseconds',
|
||||||
|
dimensions: { percentile: 'p99' },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
// Emit latency metrics
|
// Emit error metric for 5xx (separate call since it has different dimensions)
|
||||||
await emitLatency(SERVICE_NAME, responseTime, 'p50');
|
|
||||||
await emitLatency(SERVICE_NAME, responseTime, 'p95');
|
|
||||||
await emitLatency(SERVICE_NAME, responseTime, 'p99');
|
|
||||||
|
|
||||||
// Emit error metric for 5xx
|
|
||||||
if (statusCode >= 500) {
|
if (statusCode >= 500) {
|
||||||
await emitError(SERVICE_NAME, 'server_error');
|
await emitError(SERVICE_NAME, 'server_error');
|
||||||
fastify.log.warn({
|
fastify.log.warn({
|
||||||
@@ -31,8 +54,8 @@ export async function monitoringMiddleware(fastify: FastifyInstance) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log high latency requests (>2s)
|
// Log high latency requests (>2s) — only when not already logged as error
|
||||||
if (responseTime > 2000) {
|
else if (responseTime > 2000) {
|
||||||
fastify.log.warn({
|
fastify.log.warn({
|
||||||
event: 'high_latency',
|
event: 'high_latency',
|
||||||
method,
|
method,
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
// dd-trace must be initialized before any other module is loaded for auto-instrumentation
|
||||||
|
import '@shieldai/monitoring/datadog-init';
|
||||||
import Fastify from "fastify";
|
import Fastify from "fastify";
|
||||||
import cors from "@fastify/cors";
|
import cors from "@fastify/cors";
|
||||||
import helmet from "@fastify/helmet";
|
import helmet from "@fastify/helmet";
|
||||||
@@ -11,13 +13,9 @@ import { darkwatchRoutes } from "./routes/darkwatch.routes";
|
|||||||
import { voiceprintRoutes } from "./routes/voiceprint.routes";
|
import { voiceprintRoutes } from "./routes/voiceprint.routes";
|
||||||
import { correlationRoutes } from "./routes/correlation.routes";
|
import { correlationRoutes } from "./routes/correlation.routes";
|
||||||
import { extensionRoutes } from "./routes/extension.routes";
|
import { extensionRoutes } from "./routes/extension.routes";
|
||||||
import { initDatadog, initSentry, initDatadogLogs, captureSentryError } from "@shieldai/monitoring";
|
import { captureSentryError } from "@shieldai/monitoring";
|
||||||
import { getCorsOrigins } from "./config/api.config";
|
import { getCorsOrigins } from "./config/api.config";
|
||||||
|
|
||||||
initDatadog();
|
|
||||||
initSentry();
|
|
||||||
initDatadogLogs();
|
|
||||||
|
|
||||||
const app = Fastify({
|
const app = Fastify({
|
||||||
logger: {
|
logger: {
|
||||||
level: process.env.LOG_LEVEL || "info",
|
level: process.env.LOG_LEVEL || "info",
|
||||||
|
|||||||
@@ -52,6 +52,25 @@ enum UserRole {
|
|||||||
support
|
support
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum DetectionVerdict {
|
||||||
|
NATURAL
|
||||||
|
SYNTHETIC
|
||||||
|
UNCERTAIN
|
||||||
|
}
|
||||||
|
|
||||||
|
enum AnalysisType {
|
||||||
|
SYNTHETIC_DETECTION
|
||||||
|
VOICE_MATCH
|
||||||
|
BATCH
|
||||||
|
}
|
||||||
|
|
||||||
|
enum AnalysisJobStatus {
|
||||||
|
PENDING
|
||||||
|
RUNNING
|
||||||
|
COMPLETED
|
||||||
|
FAILED
|
||||||
|
}
|
||||||
|
|
||||||
model Account {
|
model Account {
|
||||||
id String @id @default(uuid())
|
id String @id @default(uuid())
|
||||||
userId String
|
userId String
|
||||||
@@ -337,6 +356,44 @@ model VoiceAnalysis {
|
|||||||
@@index([audioHash])
|
@@index([audioHash])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model AnalysisJob {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
userId String
|
||||||
|
analysisType AnalysisType
|
||||||
|
audioFilePath String
|
||||||
|
status AnalysisJobStatus
|
||||||
|
errorMessage String?
|
||||||
|
completedAt DateTime?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
user User @relation(fields: [userId], references: [id])
|
||||||
|
result AnalysisResult?
|
||||||
|
|
||||||
|
@@index([userId])
|
||||||
|
@@index([status])
|
||||||
|
@@index([createdAt])
|
||||||
|
}
|
||||||
|
|
||||||
|
model AnalysisResult {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
analysisJobId String
|
||||||
|
syntheticScore Float
|
||||||
|
verdict DetectionVerdict
|
||||||
|
confidence Float
|
||||||
|
processingTimeMs Int
|
||||||
|
matchedEnrollmentId String?
|
||||||
|
matchedSimilarity Float?
|
||||||
|
modelVersion String?
|
||||||
|
|
||||||
|
analysisJob AnalysisJob @relation(fields: [analysisJobId], references: [id])
|
||||||
|
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
@@index([analysisJobId])
|
||||||
|
@@index([syntheticScore])
|
||||||
|
@@index([verdict])
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================
|
// ============================================
|
||||||
// SpamShield Models (Spam Detection)
|
// SpamShield Models (Spam Detection)
|
||||||
// ============================================
|
// ============================================
|
||||||
|
|||||||
@@ -38,7 +38,7 @@
|
|||||||
{
|
{
|
||||||
"id": 5,
|
"id": 5,
|
||||||
"priority": 2,
|
"priority": 2,
|
||||||
"action": { "type": "REDIRECT", "redirect": { "urlFilter": "chrome-extension://__MSG_@@extension_id__/popup.html" } },
|
"action": { "type": "redirect", "redirect": { "url": "chrome-extension://__MSG_@@extension_id__/popup.html" } },
|
||||||
"condition": {
|
"condition": {
|
||||||
"urlFilter": "*://*.tk/*",
|
"urlFilter": "*://*.tk/*",
|
||||||
"resourceTypes": ["main_frame"],
|
"resourceTypes": ["main_frame"],
|
||||||
@@ -48,7 +48,7 @@
|
|||||||
{
|
{
|
||||||
"id": 6,
|
"id": 6,
|
||||||
"priority": 2,
|
"priority": 2,
|
||||||
"action": { "type": "REDIRECT", "redirect": { "urlFilter": "chrome-extension://__MSG_@@extension_id__/popup.html" } },
|
"action": { "type": "redirect", "redirect": { "url": "chrome-extension://__MSG_@@extension_id__/popup.html" } },
|
||||||
"condition": {
|
"condition": {
|
||||||
"urlFilter": "*://*.xyz/*",
|
"urlFilter": "*://*.xyz/*",
|
||||||
"resourceTypes": ["main_frame"],
|
"resourceTypes": ["main_frame"],
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ chrome.runtime.onInstalled.addListener(async () => {
|
|||||||
chrome.declarativeNetRequest.onRuleMatchedDebug.addListener((details) => {
|
chrome.declarativeNetRequest.onRuleMatchedDebug.addListener((details) => {
|
||||||
chrome.storage.local.get('blockedRequests').then((data) => {
|
chrome.storage.local.get('blockedRequests').then((data) => {
|
||||||
const blocked = data.blockedRequests || [];
|
const blocked = data.blockedRequests || [];
|
||||||
blocked.push({ ruleId: details.ruleId, url: details.requestUrl, timestamp: Date.now() });
|
blocked.push({ ruleId: details.rule?.ruleId || 0, url: details.request?.url || '', timestamp: Date.now() });
|
||||||
if (blocked.length > 100) blocked.shift();
|
if (blocked.length > 100) blocked.shift();
|
||||||
chrome.storage.local.set({ blockedRequests: blocked });
|
chrome.storage.local.set({ blockedRequests: blocked });
|
||||||
});
|
});
|
||||||
@@ -207,7 +207,18 @@ async function handleMessage(
|
|||||||
return { settings: await settingsManager.update(message.payload as Partial<ExtensionSettings>) };
|
return { settings: await settingsManager.update(message.payload as Partial<ExtensionSettings>) };
|
||||||
|
|
||||||
case MessageType.REPORT_PHISHING: {
|
case MessageType.REPORT_PHISHING: {
|
||||||
const report = message.payload as PhishingReport;
|
const payload = message.payload as Record<string, unknown> | undefined;
|
||||||
|
if (!payload || typeof payload.url !== 'string' || typeof payload.pageTitle !== 'string') {
|
||||||
|
return { success: false, error: 'Missing url or pageTitle' };
|
||||||
|
}
|
||||||
|
const report: PhishingReport = {
|
||||||
|
url: payload.url,
|
||||||
|
pageTitle: payload.pageTitle,
|
||||||
|
tabId: (payload.tabId as number) || 0,
|
||||||
|
timestamp: (payload.timestamp as number) || Date.now(),
|
||||||
|
reason: (payload.reason as string) || 'Manual report',
|
||||||
|
heuristics: (payload.heuristics as Record<string, unknown>) || {},
|
||||||
|
};
|
||||||
const success = await shieldApiClient.submitPhishingReport(report);
|
const success = await shieldApiClient.submitPhishingReport(report);
|
||||||
return { success };
|
return { success };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ export class UrlCache {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async loadFromStorage(): Promise<void> {
|
async loadFromStorage(): Promise<void> {
|
||||||
const data = await chrome.storage.local.get('urlCache');
|
const data = await chrome.storage.local.get('urlCache') as { urlCache: Record<string, { result: UrlCheckResult; expiresAt: number }> };
|
||||||
if (data.urlCache) {
|
if (data.urlCache) {
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
for (const [key, entry] of Object.entries(data.urlCache)) {
|
for (const [key, entry] of Object.entries(data.urlCache)) {
|
||||||
|
|||||||
@@ -1,43 +1,59 @@
|
|||||||
import { describe, it, expect } from 'vitest';
|
import { describe, it, expect, beforeEach } from 'vitest';
|
||||||
import { phishingDetector } from '../src/lib/phishing-detector';
|
import { urlCache } from '../src/lib/cache';
|
||||||
import { UrlVerdict, ThreatType } from '../src/types';
|
import { UrlCheckResult, UrlVerdict } from '../src/types';
|
||||||
|
|
||||||
describe('PhishingDetector (cache test)', () => {
|
describe('UrlCache', () => {
|
||||||
|
const sampleResult: UrlCheckResult = {
|
||||||
|
url: 'https://example.com',
|
||||||
|
domain: 'example.com',
|
||||||
|
verdict: UrlVerdict.SAFE,
|
||||||
|
confidence: 0.95,
|
||||||
|
threats: [],
|
||||||
|
cached: false,
|
||||||
|
latencyMs: 50,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
};
|
||||||
|
|
||||||
describe('analyzeUrl', () => {
|
beforeEach(async () => {
|
||||||
it('should return SAFE for legitimate URLs', () => {
|
urlCache.clear();
|
||||||
const result = phishingDetector.analyzeUrl('https://www.google.com/search?q=test');
|
});
|
||||||
expect(result.verdict).toBe(UrlVerdict.SAFE);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should detect suspicious TLD', () => {
|
it('should return null for missing URL', async () => {
|
||||||
const result = phishingDetector.analyzeUrl('https://free-prize.tk/claim');
|
const result = await urlCache.get('https://missing.com');
|
||||||
expect(result.threats.some((t) => t.type === ThreatType.DOMAIN_AGE)).toBe(true);
|
expect(result).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should detect typosquatting', () => {
|
it('should store and retrieve cached result', async () => {
|
||||||
const result = phishingDetector.analyzeUrl('https://goggle.com/login');
|
await urlCache.set('https://example.com', sampleResult);
|
||||||
expect(result.threats.some((t) => t.type === ThreatType.TYPOSQUAT)).toBe(true);
|
const cached = await urlCache.get('https://example.com');
|
||||||
});
|
expect(cached).not.toBeNull();
|
||||||
|
expect(cached!.cached).toBe(true);
|
||||||
|
expect(cached!.verdict).toBe(UrlVerdict.SAFE);
|
||||||
|
});
|
||||||
|
|
||||||
it('should detect IP address hostname', () => {
|
it('should normalize URLs by stripping hash and search', async () => {
|
||||||
const result = phishingDetector.analyzeUrl('http://192.168.1.100/admin');
|
await urlCache.set('https://example.com/page?foo=bar#section', sampleResult);
|
||||||
expect(result.threats.some((t) => t.type === ThreatType.PHISHING_HEURISTIC)).toBe(true);
|
const cached = await urlCache.get('https://example.com/page');
|
||||||
});
|
expect(cached).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it('should detect phishing pattern in hostname', () => {
|
it('should persist and restore from storage', async () => {
|
||||||
const result = phishingDetector.analyzeUrl('https://login-secure-portal.xyz/account');
|
await urlCache.set('https://test.com', sampleResult);
|
||||||
expect(result.threats.some((t) => t.type === ThreatType.PHISHING_HEURISTIC)).toBe(true);
|
await urlCache.persistToStorage();
|
||||||
});
|
urlCache.clear();
|
||||||
|
await urlCache.loadFromStorage();
|
||||||
|
const cached = await urlCache.get('https://test.com');
|
||||||
|
expect(cached).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it('should detect HTTP protocol', () => {
|
it('should evict oldest entry when at max capacity', async () => {
|
||||||
const result = phishingDetector.analyzeUrl('http://example.com/login');
|
const stats = urlCache.getStats();
|
||||||
expect(result.threats.some((t) => t.type === ThreatType.MIXED_CONTENT)).toBe(true);
|
expect(stats.max).toBe(5000);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should return UNKNOWN for malformed URLs', () => {
|
it('should handle malformed URLs gracefully', async () => {
|
||||||
const result = phishingDetector.analyzeUrl('not-a-real-url');
|
await urlCache.set('not-a-url', sampleResult);
|
||||||
expect(result.verdict).toBe(UrlVerdict.UNKNOWN);
|
const cached = await urlCache.get('not-a-url');
|
||||||
});
|
expect(cached).not.toBeNull();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
28
packages/extension/tests/setup.ts
Normal file
28
packages/extension/tests/setup.ts
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
const mockStorage: Record<string, unknown> = {};
|
||||||
|
|
||||||
|
const chromeMock = {
|
||||||
|
storage: {
|
||||||
|
local: {
|
||||||
|
set: async (data: Record<string, unknown>) => {
|
||||||
|
Object.assign(mockStorage, data);
|
||||||
|
},
|
||||||
|
get: async (key: string | string[]) => {
|
||||||
|
if (Array.isArray(key)) {
|
||||||
|
const result: Record<string, unknown> = {};
|
||||||
|
for (const k of key) result[k] = mockStorage[k];
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
return { [key]: mockStorage[key] };
|
||||||
|
},
|
||||||
|
remove: async (key: string | string[]) => {
|
||||||
|
const keys = Array.isArray(key) ? key : [key];
|
||||||
|
for (const k of keys) delete mockStorage[k];
|
||||||
|
},
|
||||||
|
clear: async () => {
|
||||||
|
Object.keys(mockStorage).forEach((k) => delete mockStorage[k]);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
(global as any).chrome = chromeMock;
|
||||||
@@ -5,5 +5,6 @@ export default defineConfig({
|
|||||||
globals: true,
|
globals: true,
|
||||||
environment: 'node',
|
environment: 'node',
|
||||||
include: ['tests/**/*.test.ts'],
|
include: ['tests/**/*.test.ts'],
|
||||||
|
setupFiles: ['./tests/setup.ts'],
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
"typescript": "^5.7.0"
|
"typescript": "^5.7.0"
|
||||||
},
|
},
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./src/index.ts"
|
".": "./src/index.ts",
|
||||||
|
"./datadog-init": "./src/datadog-init.ts"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -62,6 +62,35 @@ export async function emitMetric(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function emitBatchMetrics(metrics: {
|
||||||
|
serviceName: string;
|
||||||
|
data: { metricName: string; value: number; unit: StandardUnit; dimensions?: Record<string, string> }[];
|
||||||
|
}) {
|
||||||
|
const cw = getClient();
|
||||||
|
if (!cw) return;
|
||||||
|
|
||||||
|
const metricData = metrics.data.map((m) => ({
|
||||||
|
MetricName: m.metricName,
|
||||||
|
Dimensions: [
|
||||||
|
{ Name: 'service', Value: metrics.serviceName },
|
||||||
|
...(m.dimensions ? Object.entries(m.dimensions).map(([n, v]) => ({ Name: n, Value: v })) : []),
|
||||||
|
],
|
||||||
|
Value: m.value,
|
||||||
|
Unit: m.unit,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const command = new PutMetricDataCommand({
|
||||||
|
Namespace: NAMESPACE,
|
||||||
|
MetricData: metricData,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await cw.send(command);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('[CloudWatch] Batch metric emit failed:', (err as Error).message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function emitLatency(
|
export async function emitLatency(
|
||||||
serviceName: string,
|
serviceName: string,
|
||||||
latencyMs: number,
|
latencyMs: number,
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ const monitoringEnvSchema = z.object({
|
|||||||
DD_TRACE_ENABLED: z.string().default('true'),
|
DD_TRACE_ENABLED: z.string().default('true'),
|
||||||
DD_TRACE_SAMPLE_RATE: z.string().transform((v) => Number(v)).default('1.0'),
|
DD_TRACE_SAMPLE_RATE: z.string().transform((v) => Number(v)).default('1.0'),
|
||||||
DD_LOGS_INJECTION: z.string().default('true'),
|
DD_LOGS_INJECTION: z.string().default('true'),
|
||||||
|
DD_API_KEY: z.string().default(''),
|
||||||
|
DD_SITE: z.string().default('datadoghq.com'),
|
||||||
DD_AGENT_HOST: z.string().default('localhost'),
|
DD_AGENT_HOST: z.string().default('localhost'),
|
||||||
DD_AGENT_PORT: z.string().transform((v) => Number(v)).default('8126'),
|
DD_AGENT_PORT: z.string().transform((v) => Number(v)).default('8126'),
|
||||||
SENTRY_DSN: z.string().default(''),
|
SENTRY_DSN: z.string().default(''),
|
||||||
@@ -25,6 +27,8 @@ export function getMonitoringConfig(): MonitoringConfig {
|
|||||||
DD_TRACE_ENABLED: process.env.DD_TRACE_ENABLED,
|
DD_TRACE_ENABLED: process.env.DD_TRACE_ENABLED,
|
||||||
DD_TRACE_SAMPLE_RATE: process.env.DD_TRACE_SAMPLE_RATE,
|
DD_TRACE_SAMPLE_RATE: process.env.DD_TRACE_SAMPLE_RATE,
|
||||||
DD_LOGS_INJECTION: process.env.DD_LOGS_INJECTION,
|
DD_LOGS_INJECTION: process.env.DD_LOGS_INJECTION,
|
||||||
|
DD_API_KEY: process.env.DD_API_KEY,
|
||||||
|
DD_SITE: process.env.DD_SITE,
|
||||||
DD_AGENT_HOST: process.env.DD_AGENT_HOST,
|
DD_AGENT_HOST: process.env.DD_AGENT_HOST,
|
||||||
DD_AGENT_PORT: process.env.DD_AGENT_PORT,
|
DD_AGENT_PORT: process.env.DD_AGENT_PORT,
|
||||||
SENTRY_DSN: process.env.SENTRY_DSN,
|
SENTRY_DSN: process.env.SENTRY_DSN,
|
||||||
|
|||||||
8
packages/monitoring/src/datadog-init.ts
Normal file
8
packages/monitoring/src/datadog-init.ts
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
import { getMonitoringConfig } from './config';
|
||||||
|
import { initDatadog } from './datadog';
|
||||||
|
import { initSentry } from './sentry';
|
||||||
|
import { initDatadogLogs } from './datadog-logs';
|
||||||
|
|
||||||
|
initDatadog();
|
||||||
|
initSentry();
|
||||||
|
initDatadogLogs();
|
||||||
@@ -24,7 +24,7 @@ export function initDatadogLogs() {
|
|||||||
service,
|
service,
|
||||||
});
|
});
|
||||||
|
|
||||||
await fetch(`${logIntakeUrl}/api/v2/logs`, {
|
const response = await fetch(`${logIntakeUrl}/api/v2/logs`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'DD-API-KEY': process.env.DD_API_KEY!,
|
'DD-API-KEY': process.env.DD_API_KEY!,
|
||||||
@@ -32,6 +32,12 @@ export function initDatadogLogs() {
|
|||||||
},
|
},
|
||||||
body: payload,
|
body: payload,
|
||||||
});
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
console.warn(
|
||||||
|
`[Datadog Logs] HTTP ${response.status} response from intake API`,
|
||||||
|
await response.text()
|
||||||
|
);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.warn('[Datadog Logs] Forward failed:', (err as Error).message);
|
console.warn('[Datadog Logs] Forward failed:', (err as Error).message);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ export function setSentryContext(name: string, data: Record<string, unknown>) {
|
|||||||
export function getSentryHub() {
|
export function getSentryHub() {
|
||||||
try {
|
try {
|
||||||
const Sentry = require('@sentry/node');
|
const Sentry = require('@sentry/node');
|
||||||
return Sentry.getCurrentHub?.() || Sentry.hub;
|
return Sentry.getCurrentScope?.() || Sentry.getCurrentHub?.() || Sentry.hub;
|
||||||
} catch {
|
} catch {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
20
pnpm-lock.yaml
generated
20
pnpm-lock.yaml
generated
@@ -490,9 +490,15 @@ importers:
|
|||||||
'@shieldai/types':
|
'@shieldai/types':
|
||||||
specifier: workspace:*
|
specifier: workspace:*
|
||||||
version: link:../../packages/types
|
version: link:../../packages/types
|
||||||
|
'@types/uuid':
|
||||||
|
specifier: ^11.0.0
|
||||||
|
version: 11.0.0
|
||||||
node-cache:
|
node-cache:
|
||||||
specifier: ^5.1.2
|
specifier: ^5.1.2
|
||||||
version: 5.1.2
|
version: 5.1.2
|
||||||
|
uuid:
|
||||||
|
specifier: ^14.0.0
|
||||||
|
version: 14.0.0
|
||||||
devDependencies:
|
devDependencies:
|
||||||
'@vitest/coverage-v8':
|
'@vitest/coverage-v8':
|
||||||
specifier: ^4.1.5
|
specifier: ^4.1.5
|
||||||
@@ -2748,6 +2754,10 @@ packages:
|
|||||||
'@types/tough-cookie@4.0.5':
|
'@types/tough-cookie@4.0.5':
|
||||||
resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==}
|
resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==}
|
||||||
|
|
||||||
|
'@types/uuid@11.0.0':
|
||||||
|
resolution: {integrity: sha512-HVyk8nj2m+jcFRNazzqyVKiZezyhDKrGUA3jlEcg/nZ6Ms+qHwocba1Y/AaVaznJTAM9xpdFSh+ptbNrhOGvZA==}
|
||||||
|
deprecated: This is a stub types definition. uuid provides its own type definitions, so you do not need this installed.
|
||||||
|
|
||||||
'@types/ws@8.18.1':
|
'@types/ws@8.18.1':
|
||||||
resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
|
resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
|
||||||
|
|
||||||
@@ -5552,6 +5562,10 @@ packages:
|
|||||||
deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).
|
deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
uuid@14.0.0:
|
||||||
|
resolution: {integrity: sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==}
|
||||||
|
hasBin: true
|
||||||
|
|
||||||
uuid@8.3.2:
|
uuid@8.3.2:
|
||||||
resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
|
resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
|
||||||
deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).
|
deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).
|
||||||
@@ -8450,6 +8464,10 @@ snapshots:
|
|||||||
'@types/tough-cookie@4.0.5':
|
'@types/tough-cookie@4.0.5':
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
|
'@types/uuid@11.0.0':
|
||||||
|
dependencies:
|
||||||
|
uuid: 14.0.0
|
||||||
|
|
||||||
'@types/ws@8.18.1':
|
'@types/ws@8.18.1':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@types/node': 25.6.0
|
'@types/node': 25.6.0
|
||||||
@@ -11809,6 +11827,8 @@ snapshots:
|
|||||||
|
|
||||||
uuid@10.0.0: {}
|
uuid@10.0.0: {}
|
||||||
|
|
||||||
|
uuid@14.0.0: {}
|
||||||
|
|
||||||
uuid@8.3.2: {}
|
uuid@8.3.2: {}
|
||||||
|
|
||||||
uuid@9.0.1: {}
|
uuid@9.0.1: {}
|
||||||
|
|||||||
@@ -10,14 +10,16 @@
|
|||||||
"lint": "eslint src/"
|
"lint": "eslint src/"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@shieldai/correlation": "workspace:*",
|
||||||
"@shieldai/db": "workspace:*",
|
"@shieldai/db": "workspace:*",
|
||||||
"@shieldai/types": "workspace:*",
|
"@shieldai/types": "workspace:*",
|
||||||
"@shieldai/correlation": "workspace:*",
|
"@types/uuid": "^11.0.0",
|
||||||
"node-cache": "^5.1.2"
|
"node-cache": "^5.1.2",
|
||||||
|
"uuid": "^14.0.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"vitest": "^4.1.5",
|
"@vitest/coverage-v8": "^4.1.5",
|
||||||
"@vitest/coverage-v8": "^4.1.5"
|
"vitest": "^4.1.5"
|
||||||
},
|
},
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./src/index.ts"
|
".": "./src/index.ts"
|
||||||
|
|||||||
@@ -6,9 +6,11 @@ import {
|
|||||||
AnalysisType,
|
AnalysisType,
|
||||||
AnalysisResultOutput,
|
AnalysisResultOutput,
|
||||||
} from "@shieldai/types";
|
} from "@shieldai/types";
|
||||||
|
import { logger } from "../logger";
|
||||||
|
|
||||||
export class BatchAnalysisService {
|
export class BatchAnalysisService {
|
||||||
private analysisService: AnalysisService;
|
private analysisService: AnalysisService;
|
||||||
|
private readonly maxConcurrency = 5;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.analysisService = new AnalysisService();
|
this.analysisService = new AnalysisService();
|
||||||
@@ -19,43 +21,56 @@ export class BatchAnalysisService {
|
|||||||
userId: string
|
userId: string
|
||||||
): Promise<BatchResult> {
|
): Promise<BatchResult> {
|
||||||
const batchId = `batch_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
const batchId = `batch_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||||
|
logger.info("Starting batch analysis", { batchId, userId, totalFiles: input.audioBuffers.length });
|
||||||
const results: AnalysisResultOutput[] = [];
|
const results: AnalysisResultOutput[] = [];
|
||||||
const errors: Array<{ name: string; error: string }> = [];
|
const errors: Array<{ name: string; error: string }> = [];
|
||||||
|
|
||||||
for (const audioInput of input.audioBuffers) {
|
const processWithConcurrency = async (limit: number) => {
|
||||||
try {
|
for (let i = 0; i < input.audioBuffers.length; i += limit) {
|
||||||
const result = await this.analysisService.analyze(
|
const chunk = input.audioBuffers.slice(i, i + limit);
|
||||||
{
|
|
||||||
audioBuffer: audioInput.buffer,
|
const promises = chunk.map(async (audioInput: { name: string; buffer: Buffer; sampleRate?: number }) => {
|
||||||
sampleRate: audioInput.sampleRate,
|
try {
|
||||||
analysisType: input.analysisType || AnalysisType.SYNTHETIC_DETECTION,
|
const result = await this.analysisService.analyze(
|
||||||
},
|
{
|
||||||
userId
|
audioBuffer: audioInput.buffer,
|
||||||
);
|
sampleRate: audioInput.sampleRate,
|
||||||
results.push(result);
|
analysisType: input.analysisType || AnalysisType.SYNTHETIC_DETECTION,
|
||||||
} catch (err) {
|
},
|
||||||
const message = err instanceof Error ? err.message : "Analysis failed";
|
userId
|
||||||
errors.push({ name: audioInput.name, error: message });
|
);
|
||||||
|
return { success: true, result, name: audioInput.name };
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : "Analysis failed";
|
||||||
|
return { success: false, error: message, name: audioInput.name };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const outcomes = await Promise.allSettled(promises);
|
||||||
|
|
||||||
|
for (const outcome of outcomes) {
|
||||||
|
if (outcome.status === 'fulfilled') {
|
||||||
|
if (outcome.value.success && outcome.value.result) {
|
||||||
|
results.push(outcome.value.result);
|
||||||
|
} else if (!outcome.value.success && outcome.value.name) {
|
||||||
|
errors.push({ name: outcome.value.name, error: outcome.value.error || "Analysis failed" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
const batchJob = await prisma.analysisJob.create({
|
await processWithConcurrency(this.maxConcurrency);
|
||||||
data: {
|
|
||||||
userId,
|
logger.info("Batch analysis completed", {
|
||||||
analysisType: AnalysisType.BATCH,
|
batchId,
|
||||||
audioFilePath: `voiceprint/${userId}/${batchId}`,
|
successfulResults: results.length,
|
||||||
status: errors.length === input.audioBuffers.length
|
failedCount: errors.length
|
||||||
? AnalysisJobStatus.FAILED
|
|
||||||
: AnalysisJobStatus.COMPLETED,
|
|
||||||
errorMessage:
|
|
||||||
errors.length > 0 ? `${errors.length} of ${input.audioBuffers.length} files failed` : undefined,
|
|
||||||
completedAt: new Date(),
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
batchId,
|
batchId,
|
||||||
jobId: batchJob.id,
|
jobId: `batch_${batchId}`,
|
||||||
totalFiles: input.audioBuffers.length,
|
totalFiles: input.audioBuffers.length,
|
||||||
successfulResults: results.length,
|
successfulResults: results.length,
|
||||||
failedCount: errors.length,
|
failedCount: errors.length,
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
import { spawn } from "child_process";
|
import { spawn } from "child_process";
|
||||||
|
import { v4 as uuidv4 } from "uuid";
|
||||||
|
import { logger } from "../logger";
|
||||||
|
|
||||||
const EMBEDDING_DIM = 192;
|
const EMBEDDING_DIM = 192;
|
||||||
const MODEL_VERSION = "ecapa-tdnn-0.1.0-mock";
|
const MODEL_VERSION = "ecapa-tdnn-0.1.0-mock";
|
||||||
|
|
||||||
export class EmbeddingService {
|
export class EmbeddingService {
|
||||||
private mlServiceUrl: string;
|
private mlServiceUrl: string;
|
||||||
|
private readonly maxRetries = 3;
|
||||||
|
private readonly retryDelay = 1000;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.mlServiceUrl = process.env.VOICEPRINT_ML_URL || "http://localhost:8001";
|
this.mlServiceUrl = process.env.VOICEPRINT_ML_URL || "http://localhost:8001";
|
||||||
@@ -14,20 +18,34 @@ export class EmbeddingService {
|
|||||||
const mlAvailable = await this.checkMLService();
|
const mlAvailable = await this.checkMLService();
|
||||||
|
|
||||||
if (mlAvailable) {
|
if (mlAvailable) {
|
||||||
|
logger.info("Using ML service for embedding extraction", { mlUrl: this.mlServiceUrl });
|
||||||
return this.extractViaML(audioBuffer);
|
return this.extractViaML(audioBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.extractMock(audioBuffer);
|
logger.info("Using mock embedding generation", { audioBufferLength: audioBuffer.length });
|
||||||
|
return this.generateMockFromBuffer(audioBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
async classify(embedding: number[]): Promise<number> {
|
async classify(embedding: number[]): Promise<number> {
|
||||||
const mlAvailable = await this.checkMLService();
|
const mlAvailable = await this.checkMLService();
|
||||||
|
|
||||||
if (mlAvailable) {
|
if (mlAvailable) {
|
||||||
|
logger.info("Using ML service for classification", { embeddingLength: embedding.length });
|
||||||
return this.classifyViaML(embedding);
|
return this.classifyViaML(embedding);
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.classifyMock(embedding);
|
logger.info("Using mock classification", { embeddingLength: embedding.length });
|
||||||
|
const mean = embedding.reduce((s, v) => s + v, 0) / embedding.length;
|
||||||
|
const variance = embedding.reduce((s, v) => s + (v - mean) ** 2, 0) / embedding.length;
|
||||||
|
const stdDev = Math.sqrt(variance);
|
||||||
|
|
||||||
|
const syntheticIndicators = [
|
||||||
|
stdDev < 0.1 ? 0.8 : 0.2,
|
||||||
|
Math.abs(mean) > 0.5 ? 0.7 : 0.3,
|
||||||
|
this.hasArtifacts(embedding) ? 0.9 : 0.1,
|
||||||
|
];
|
||||||
|
|
||||||
|
return syntheticIndicators.reduce((s, v) => s + v, 0) / syntheticIndicators.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
getModelVersion(): string {
|
getModelVersion(): string {
|
||||||
@@ -105,26 +123,29 @@ except:
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async extractMock(audioBuffer: Buffer): Promise<EmbeddingOutput> {
|
private hasArtifacts(embedding: number[]): boolean {
|
||||||
return this.generateMockFromBuffer(audioBuffer);
|
const window = 16;
|
||||||
}
|
let artifactCount = 0;
|
||||||
|
|
||||||
private async classifyMock(embedding: number[]): Promise<number> {
|
for (let i = 0; i < embedding.length - window; i += window) {
|
||||||
const mean = embedding.reduce((s, v) => s + v, 0) / embedding.length;
|
const slice = embedding.slice(i, i + window);
|
||||||
const variance = embedding.reduce((s, v) => s + (v - mean) ** 2, 0) / embedding.length;
|
const localMean = slice.reduce((s, v) => s + v, 0) / slice.length;
|
||||||
const stdDev = Math.sqrt(variance);
|
const localVar = slice.reduce((s, v) => s + (v - localMean) ** 2, 0) / slice.length;
|
||||||
|
|
||||||
const syntheticIndicators = [
|
if (localVar < 0.001) artifactCount++;
|
||||||
stdDev < 0.1 ? 0.8 : 0.2,
|
}
|
||||||
Math.abs(mean) > 0.5 ? 0.7 : 0.3,
|
|
||||||
this.hasArtifacts(embedding) ? 0.9 : 0.1,
|
|
||||||
];
|
|
||||||
|
|
||||||
return syntheticIndicators.reduce((s, v) => s + v, 0) / syntheticIndicators.length;
|
return artifactCount > embedding.length / window / 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
private generateMockFromBuffer(audioBuffer: Buffer): EmbeddingOutput {
|
private generateMockFromBuffer(audioBuffer: Buffer): EmbeddingOutput {
|
||||||
const seed = this.computeSeed(audioBuffer);
|
let hash = 0;
|
||||||
|
const sampleSize = Math.min(audioBuffer.length, 1024);
|
||||||
|
for (let i = 0; i < sampleSize; i += 4) {
|
||||||
|
hash = ((hash << 5) - hash + audioBuffer.readInt32LE(i)) | 0;
|
||||||
|
}
|
||||||
|
const seed = Math.abs(hash);
|
||||||
|
|
||||||
const rng = this.createRNG(seed);
|
const rng = this.createRNG(seed);
|
||||||
const vector: number[] = [];
|
const vector: number[] = [];
|
||||||
|
|
||||||
@@ -141,22 +162,8 @@ except:
|
|||||||
return { vector: normalized, dimension: EMBEDDING_DIM };
|
return { vector: normalized, dimension: EMBEDDING_DIM };
|
||||||
}
|
}
|
||||||
|
|
||||||
private hasArtifacts(embedding: number[]): boolean {
|
|
||||||
const window = 16;
|
|
||||||
let artifactCount = 0;
|
|
||||||
|
|
||||||
for (let i = 0; i < embedding.length - window; i += window) {
|
|
||||||
const slice = embedding.slice(i, i + window);
|
|
||||||
const localMean = slice.reduce((s, v) => s + v, 0) / slice.length;
|
|
||||||
const localVar = slice.reduce((s, v) => s + (v - localMean) ** 2, 0) / slice.length;
|
|
||||||
|
|
||||||
if (localVar < 0.001) artifactCount++;
|
|
||||||
}
|
|
||||||
|
|
||||||
return artifactCount > embedding.length / window / 3;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async checkMLService(): Promise<boolean> {
|
private async checkMLService(): Promise<boolean> {
|
||||||
|
logger.info("Checking ML service availability", { mlUrl: this.mlServiceUrl });
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
const proc = spawn("python3", [
|
const proc = spawn("python3", [
|
||||||
"-c",
|
"-c",
|
||||||
@@ -173,15 +180,6 @@ except:
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private computeSeed(buffer: Buffer): number {
|
|
||||||
let hash = 0;
|
|
||||||
const sampleSize = Math.min(buffer.length, 1024);
|
|
||||||
for (let i = 0; i < sampleSize; i += 4) {
|
|
||||||
hash = ((hash << 5) - hash + buffer.readInt32LE(i)) | 0;
|
|
||||||
}
|
|
||||||
return Math.abs(hash);
|
|
||||||
}
|
|
||||||
|
|
||||||
private createRNG(seed: number): () => number {
|
private createRNG(seed: number): () => number {
|
||||||
return () => {
|
return () => {
|
||||||
seed = (seed * 1664525 + 1013904223) & 0xffffffff;
|
seed = (seed * 1664525 + 1013904223) & 0xffffffff;
|
||||||
|
|||||||
@@ -23,11 +23,13 @@ export class VoiceEnrollmentService {
|
|||||||
const enrollment = await prisma.voiceEnrollment.create({
|
const enrollment = await prisma.voiceEnrollment.create({
|
||||||
data: {
|
data: {
|
||||||
userId,
|
userId,
|
||||||
label: input.label,
|
name: input.label,
|
||||||
embeddingVector: embedding.vector,
|
voiceHash: this.computeVoiceHash(embedding.vector),
|
||||||
embeddingDim: embedding.dimension,
|
audioMetadata: {
|
||||||
sampleRate: preprocessed.sampleRate,
|
sampleRate: preprocessed.sampleRate,
|
||||||
durationSec: preprocessed.durationSec,
|
durationSec: preprocessed.durationSec,
|
||||||
|
embeddingDim: embedding.dimension,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -35,10 +37,10 @@ export class VoiceEnrollmentService {
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
id: enrollment.id,
|
id: enrollment.id,
|
||||||
label: enrollment.label,
|
label: enrollment.name,
|
||||||
embeddingDim: enrollment.embeddingDim,
|
embeddingDim: preprocessed.sampleRate,
|
||||||
sampleRate: enrollment.sampleRate,
|
sampleRate: preprocessed.sampleRate,
|
||||||
durationSec: enrollment.durationSec,
|
durationSec: preprocessed.durationSec,
|
||||||
createdAt: enrollment.createdAt,
|
createdAt: enrollment.createdAt,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
36
services/voiceprint/src/logger.ts
Normal file
36
services/voiceprint/src/logger.ts
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
import { FastifyLoggerOptions } from 'fastify';
|
||||||
|
|
||||||
|
export interface Logger {
|
||||||
|
info(message: string, context?: Record<string, unknown>): void;
|
||||||
|
warn(message: string, context?: Record<string, unknown>): void;
|
||||||
|
error(message: string, context?: Record<string, unknown>): void;
|
||||||
|
debug(message: string, context?: Record<string, unknown>): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ConsoleLogger implements Logger {
|
||||||
|
info(message: string, context?: Record<string, unknown>): void {
|
||||||
|
const timestamp = new Date().toISOString();
|
||||||
|
const logContext = context ? ` ${JSON.stringify(context)}` : '';
|
||||||
|
console.log(`[${timestamp}] [INFO] ${message}${logContext}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
warn(message: string, context?: Record<string, unknown>): void {
|
||||||
|
const timestamp = new Date().toISOString();
|
||||||
|
const logContext = context ? ` ${JSON.stringify(context)}` : '';
|
||||||
|
console.warn(`[${timestamp}] [WARN] ${message}${logContext}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
error(message: string, context?: Record<string, unknown>): void {
|
||||||
|
const timestamp = new Date().toISOString();
|
||||||
|
const logContext = context ? ` ${JSON.stringify(context)}` : '';
|
||||||
|
console.error(`[${timestamp}] [ERROR] ${message}${logContext}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
debug(message: string, context?: Record<string, unknown>): void {
|
||||||
|
const timestamp = new Date().toISOString();
|
||||||
|
const logContext = context ? ` ${JSON.stringify(context)}` : '';
|
||||||
|
console.debug(`[${timestamp}] [DEBUG] ${message}${logContext}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const logger = new ConsoleLogger();
|
||||||
@@ -1,3 +1,14 @@
|
|||||||
|
/**
|
||||||
|
* VoicePrint Service - Legacy Module
|
||||||
|
*
|
||||||
|
* @deprecated This file contains legacy service implementations.
|
||||||
|
* Migrate to the new modular structure:
|
||||||
|
* - Use `import { AnalysisService } from './analysis/AnalysisService'` for analysis
|
||||||
|
* - Use `import { BatchAnalysisService } from './analysis/BatchAnalysisService'` for batch operations
|
||||||
|
* - Use `import { EmbeddingService } from './embedding/EmbeddingService'` for embeddings
|
||||||
|
* - Use `import { VoiceEnrollmentService } from './enrollment/VoiceEnrollmentService'` for enrollment
|
||||||
|
*/
|
||||||
|
|
||||||
import { prisma, VoiceEnrollment, VoiceAnalysis } from '@shieldai/db';
|
import { prisma, VoiceEnrollment, VoiceAnalysis } from '@shieldai/db';
|
||||||
import {
|
import {
|
||||||
voicePrintEnv,
|
voicePrintEnv,
|
||||||
@@ -8,6 +19,8 @@ import {
|
|||||||
voicePrintFeatureFlags,
|
voicePrintFeatureFlags,
|
||||||
} from './voiceprint.config';
|
} from './voiceprint.config';
|
||||||
import { checkFlag } from './voiceprint.feature-flags';
|
import { checkFlag } from './voiceprint.feature-flags';
|
||||||
|
import { createHash } from 'crypto';
|
||||||
|
import { logger } from './logger';
|
||||||
|
|
||||||
// Audio preprocessing service
|
// Audio preprocessing service
|
||||||
export class AudioPreprocessor {
|
export class AudioPreprocessor {
|
||||||
@@ -197,12 +210,10 @@ export class VoiceEnrollmentService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private computeEmbeddingHash(embedding: number[]): string {
|
private computeEmbeddingHash(embedding: number[]): string {
|
||||||
let hash = 0;
|
const hash = createHash('sha256')
|
||||||
for (let i = 0; i < embedding.length; i++) {
|
.update(JSON.stringify(embedding))
|
||||||
hash = ((hash << 5) - hash) + embedding[i];
|
.digest('hex');
|
||||||
hash |= 0;
|
return `vp_${hash.substring(0, 16)}_${embedding.length}`;
|
||||||
}
|
|
||||||
return `vp_${Math.abs(hash).toString(16)}_${embedding.length}`;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -287,13 +298,10 @@ export class AnalysisService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private computeAudioHash(buffer: Buffer): string {
|
private computeAudioHash(buffer: Buffer): string {
|
||||||
let hash = 0;
|
const hash = createHash('sha256')
|
||||||
const sampleSize = Math.min(buffer.length, 1024);
|
.update(buffer)
|
||||||
for (let i = 0; i < sampleSize; i += 8) {
|
.digest('hex');
|
||||||
hash = ((hash << 5) - hash) + buffer.readUInt8(i);
|
return `audio_${hash.substring(0, 16)}`;
|
||||||
hash |= 0;
|
|
||||||
}
|
|
||||||
return `audio_${Math.abs(hash).toString(16)}`;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -328,31 +336,66 @@ export class BatchAnalysisService {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const jobId = `batch_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
||||||
|
logger.info('Starting batch analysis', { jobId, userId, fileCount: files.length });
|
||||||
|
|
||||||
const analysisService = new AnalysisService();
|
const analysisService = new AnalysisService();
|
||||||
const results: VoiceAnalysis[] = [];
|
const results: VoiceAnalysis[] = [];
|
||||||
let synthetic = 0;
|
let synthetic = 0;
|
||||||
let natural = 0;
|
let natural = 0;
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
|
|
||||||
for (const file of files) {
|
// Process with concurrency control
|
||||||
try {
|
const concurrencyLimit = 5;
|
||||||
const result = await analysisService.analyze(userId, file.buffer, {
|
for (let i = 0; i < files.length; i += concurrencyLimit) {
|
||||||
enrollmentId: options?.enrollmentId,
|
const chunk = files.slice(i, i + concurrencyLimit);
|
||||||
audioUrl: file.audioUrl,
|
const promises = chunk.map(async (file) => {
|
||||||
});
|
try {
|
||||||
results.push(result);
|
const result = await analysisService.analyze(userId, file.buffer, {
|
||||||
if (result.isSynthetic) {
|
enrollmentId: options?.enrollmentId,
|
||||||
synthetic++;
|
audioUrl: file.audioUrl,
|
||||||
} else {
|
});
|
||||||
natural++;
|
return { success: true as const, result, name: file.name };
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Batch analysis failed for file', { fileName: file.name, jobId, error });
|
||||||
|
return { success: false as const, error: error instanceof Error ? error.message : 'Unknown error', name: file.name };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const outcomes = await Promise.allSettled(promises);
|
||||||
|
for (const outcome of outcomes) {
|
||||||
|
if (outcome.status === 'fulfilled') {
|
||||||
|
if (outcome.value.success) {
|
||||||
|
results.push(outcome.value.result);
|
||||||
|
if (outcome.value.result.isSynthetic) {
|
||||||
|
synthetic++;
|
||||||
|
} else {
|
||||||
|
natural++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
failed++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
|
||||||
console.error(`Batch analysis failed for ${file.name}:`, error);
|
|
||||||
failed++;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const jobId = `batch_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
// Persist batch jobId to database
|
||||||
|
await prisma.$transaction([
|
||||||
|
prisma.$executeRawUnsafe('INSERT INTO batch_jobs (id, user_id, total_files, status, created_at) VALUES ($1, $2, $3, $4, NOW()) ON CONFLICT (id) DO NOTHING', jobId, userId, files.length, 'completed'),
|
||||||
|
...results.map(result =>
|
||||||
|
prisma.$executeRawUnsafe('UPDATE voice_analysis SET batch_job_id = $1 WHERE id = $2', jobId, result.id)
|
||||||
|
)
|
||||||
|
]).catch(err => {
|
||||||
|
logger.warn('Failed to persist batch jobId', { jobId, error: err instanceof Error ? err.message : String(err) });
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info('Batch analysis completed', {
|
||||||
|
jobId,
|
||||||
|
total: files.length,
|
||||||
|
synthetic,
|
||||||
|
natural,
|
||||||
|
failed
|
||||||
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
jobId,
|
jobId,
|
||||||
@@ -367,61 +410,39 @@ export class BatchAnalysisService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Embedding service — ECAPA-TDNN inference wrapper
|
// Deprecated: Use embedding/EmbeddingService.ts instead
|
||||||
|
// This class is kept for backward compatibility but delegates to the canonical service
|
||||||
|
/**
|
||||||
|
* @deprecated Use `import { EmbeddingService } from './embedding/EmbeddingService'` instead
|
||||||
|
*/
|
||||||
export class EmbeddingService {
|
export class EmbeddingService {
|
||||||
private initialized = false;
|
private initialized = false;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize the ECAPA-TDNN model.
|
* Initialize the ECAPA-TDNN model.
|
||||||
|
* @deprecated Use the canonical EmbeddingService from embedding/EmbeddingService.ts
|
||||||
*/
|
*/
|
||||||
async initialize(): Promise<void> {
|
async initialize(): Promise<void> {
|
||||||
if (this.initialized) return;
|
if (this.initialized) return;
|
||||||
|
|
||||||
// TODO: Connect to Python ML service for real inference
|
|
||||||
// const response = await fetch(`${voicePrintEnv.ML_SERVICE_URL}/initialize`, {
|
|
||||||
// method: 'POST',
|
|
||||||
// body: JSON.stringify({ modelPath: voicePrintEnv.ECAPA_TDNN_MODEL_PATH }),
|
|
||||||
// });
|
|
||||||
|
|
||||||
this.initialized = true;
|
this.initialized = true;
|
||||||
console.log('Embedding service initialized (mock model)');
|
logger.warn('Deprecated EmbeddingService initialized - migrate to embedding/EmbeddingService.ts');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extract voice embedding from audio.
|
* Extract voice embedding from audio.
|
||||||
|
* @deprecated Use the canonical EmbeddingService from embedding/EmbeddingService.ts
|
||||||
*/
|
*/
|
||||||
async extract(audioBuffer: Buffer): Promise<number[]> {
|
async extract(audioBuffer: Buffer): Promise<number[]> {
|
||||||
await this.initialize();
|
await this.initialize();
|
||||||
|
// Delegate to canonical implementation
|
||||||
// TODO: Call Python ML service
|
const canonicalService = new CanonicalEmbeddingService();
|
||||||
// const response = await fetch(`${voicePrintEnv.ML_SERVICE_URL}/embed`, {
|
const result = await canonicalService.extract(audioBuffer);
|
||||||
// method: 'POST',
|
return result.vector;
|
||||||
// body: audioBuffer,
|
|
||||||
// });
|
|
||||||
// const data = await response.json();
|
|
||||||
// return data.embedding;
|
|
||||||
|
|
||||||
// Mock: generate deterministic embedding based on buffer content
|
|
||||||
const dims = voicePrintEnv.EMBEDDING_DIMENSIONS;
|
|
||||||
const embedding: number[] = new Array(dims);
|
|
||||||
let hash = 0;
|
|
||||||
for (let i = 0; i < Math.min(audioBuffer.length, 256); i++) {
|
|
||||||
hash = ((hash << 5) - hash) + audioBuffer[i];
|
|
||||||
hash |= 0;
|
|
||||||
}
|
|
||||||
for (let i = 0; i < dims; i++) {
|
|
||||||
hash = ((hash << 5) - hash) + i;
|
|
||||||
hash |= 0;
|
|
||||||
embedding[i] = (Math.abs(hash) % 1000) / 1000.0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// L2 normalize
|
|
||||||
const norm = Math.sqrt(embedding.reduce((s, v) => s + v * v, 0));
|
|
||||||
return embedding.map((v) => v / norm);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run full analysis: embedding + synthetic detection.
|
* Run full analysis: embedding + synthetic detection.
|
||||||
|
* @deprecated Use AnalysisService from analysis/AnalysisService.ts instead
|
||||||
*/
|
*/
|
||||||
async analyze(audioBuffer: Buffer): Promise<{
|
async analyze(audioBuffer: Buffer): Promise<{
|
||||||
confidence: number;
|
confidence: number;
|
||||||
@@ -429,64 +450,92 @@ export class EmbeddingService {
|
|||||||
features: Record<string, number>;
|
features: Record<string, number>;
|
||||||
embedding: number[];
|
embedding: number[];
|
||||||
}> {
|
}> {
|
||||||
const embedding = await this.extract(audioBuffer);
|
const embeddingService = new CanonicalEmbeddingService();
|
||||||
|
const result = await embeddingService.analyze(audioBuffer);
|
||||||
// TODO: Run synthetic voice detection model
|
|
||||||
// For MVP, use heuristic based on embedding statistics
|
|
||||||
const confidence = this.estimateSyntheticConfidence(audioBuffer, embedding);
|
|
||||||
const detectionType =
|
|
||||||
confidence >= voicePrintEnv.SYNTHETIC_THRESHOLD
|
|
||||||
? DetectionType.SYNTHETIC_VOICE
|
|
||||||
: DetectionType.NATURAL;
|
|
||||||
|
|
||||||
const features = this.extractAnalysisFeatures(audioBuffer, embedding);
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
confidence,
|
confidence: result.confidence,
|
||||||
detectionType,
|
detectionType: result.detectionType,
|
||||||
features,
|
features: result.features,
|
||||||
embedding,
|
embedding: result.vector,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private estimateSyntheticConfidence(
|
// Canonical embedding service - single source of truth for embedding logic
|
||||||
buffer: Buffer,
|
class CanonicalEmbeddingService {
|
||||||
embedding: number[]
|
private initialized = false;
|
||||||
): number {
|
|
||||||
// Heuristic features for synthetic detection
|
|
||||||
const meanAmplitude =
|
|
||||||
buffer.reduce((s, v) => s + v, 0) / buffer.length / 255;
|
|
||||||
const embeddingStdDev =
|
|
||||||
Math.sqrt(
|
|
||||||
embedding.reduce((s, v) => s + (v - embedding.reduce((a, b) => a + b) / embedding.length) ** 2, 0) /
|
|
||||||
embedding.length
|
|
||||||
) || 0;
|
|
||||||
|
|
||||||
// Combine features into confidence score
|
async initialize(): Promise<void> {
|
||||||
const amplitudeScore = Math.abs(meanAmplitude - 0.5) * 2;
|
if (this.initialized) return;
|
||||||
const embeddingScore = 1.0 - Math.min(1.0, embeddingStdDev * 2);
|
this.initialized = true;
|
||||||
|
logger.info('Canonical EmbeddingService initialized', { modelVersion: 'ecapa-tdnn-v1-mock' });
|
||||||
return Math.min(
|
|
||||||
1.0,
|
|
||||||
amplitudeScore * 0.3 + embeddingScore * 0.4 + Math.random() * 0.3
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private extractAnalysisFeatures(
|
async extract(audioBuffer: Buffer): Promise<{ vector: number[]; dimension: number }> {
|
||||||
buffer: Buffer,
|
await this.initialize();
|
||||||
embedding: number[]
|
// Use the same mock generation as embedding/EmbeddingService.ts for consistency
|
||||||
): Record<string, number> {
|
const dims = voicePrintEnv.EMBEDDING_DIMENSIONS;
|
||||||
const meanAmplitude =
|
let hash = 0;
|
||||||
buffer.reduce((s, v) => s + v, 0) / buffer.length / 255;
|
const sampleSize = Math.min(audioBuffer.length, 1024);
|
||||||
const zeroCrossings = buffer.reduce((count, v, i, arr) => {
|
for (let i = 0; i < sampleSize; i += 4) {
|
||||||
|
hash = ((hash << 5) - hash + audioBuffer.readInt32LE(i)) | 0;
|
||||||
|
}
|
||||||
|
const seed = Math.abs(hash);
|
||||||
|
const rng = this.createRNG(seed);
|
||||||
|
|
||||||
|
const vector: number[] = [];
|
||||||
|
for (let i = 0; i < dims; i++) {
|
||||||
|
const u1 = rng();
|
||||||
|
const u2 = rng();
|
||||||
|
const gauss = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
|
||||||
|
vector.push(parseFloat(gauss.toFixed(6)));
|
||||||
|
}
|
||||||
|
|
||||||
|
const norm = Math.sqrt(vector.reduce((s, v) => s + v * v, 0));
|
||||||
|
const normalized = vector.map((v) => parseFloat((v / norm).toFixed(6)));
|
||||||
|
return { vector: normalized, dimension: dims };
|
||||||
|
}
|
||||||
|
|
||||||
|
async analyze(audioBuffer: Buffer): Promise<{
|
||||||
|
confidence: number;
|
||||||
|
detectionType: DetectionType;
|
||||||
|
features: Record<string, number>;
|
||||||
|
vector: number[];
|
||||||
|
}> {
|
||||||
|
const { vector } = await this.extract(audioBuffer);
|
||||||
|
|
||||||
|
// Heuristic for synthetic detection
|
||||||
|
const meanAmplitude = audioBuffer.reduce((s, v) => s + v, 0) / audioBuffer.length / 255;
|
||||||
|
const embeddingStdDev = Math.sqrt(
|
||||||
|
vector.reduce((s, v) => s + (v - vector.reduce((a, b) => a + b) / vector.length) ** 2, 0) / vector.length
|
||||||
|
) || 0;
|
||||||
|
|
||||||
|
const amplitudeScore = Math.abs(meanAmplitude - 0.5) * 2;
|
||||||
|
const embeddingScore = 1.0 - Math.min(1.0, embeddingStdDev * 2);
|
||||||
|
const confidence = Math.min(1.0, amplitudeScore * 0.3 + embeddingScore * 0.4 + Math.random() * 0.3);
|
||||||
|
|
||||||
|
const detectionType = confidence >= voicePrintEnv.SYNTHETIC_THRESHOLD
|
||||||
|
? DetectionType.SYNTHETIC_VOICE
|
||||||
|
: DetectionType.NATURAL;
|
||||||
|
|
||||||
|
const zeroCrossings = audioBuffer.reduce((count, v, i, arr) => {
|
||||||
return i > 0 && ((v - 128) * (arr[i - 1] - 128) < 0) ? count + 1 : count;
|
return i > 0 && ((v - 128) * (arr[i - 1] - 128) < 0) ? count + 1 : count;
|
||||||
}, 0);
|
}, 0);
|
||||||
|
|
||||||
return {
|
const features = {
|
||||||
mean_amplitude: meanAmplitude,
|
mean_amplitude: meanAmplitude,
|
||||||
zero_crossing_rate: zeroCrossings / buffer.length,
|
zero_crossing_rate: zeroCrossings / audioBuffer.length,
|
||||||
embedding_energy: embedding.reduce((s, v) => s + v * v, 0),
|
embedding_energy: vector.reduce((s, v) => s + v * v, 0),
|
||||||
embedding_entropy: this.calculateEntropy(embedding),
|
embedding_entropy: this.calculateEntropy(vector),
|
||||||
|
};
|
||||||
|
|
||||||
|
return { confidence, detectionType, features, vector };
|
||||||
|
}
|
||||||
|
|
||||||
|
private createRNG(seed: number): () => number {
|
||||||
|
return () => {
|
||||||
|
seed = (seed * 1664525 + 1013904223) & 0xffffffff;
|
||||||
|
return (seed >>> 0) / 0xffffffff;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -523,7 +572,7 @@ export class FAISSIndex {
|
|||||||
this.indexPath = path ?? voicePrintEnv.FAISS_INDEX_PATH;
|
this.indexPath = path ?? voicePrintEnv.FAISS_INDEX_PATH;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize or load the FAISS index.
|
* Initialize or load the FAISS index.
|
||||||
*/
|
*/
|
||||||
async initialize(): Promise<void> {
|
async initialize(): Promise<void> {
|
||||||
@@ -534,10 +583,10 @@ export class FAISSIndex {
|
|||||||
// this.index = faiss.readIndex(this.indexPath);
|
// this.index = faiss.readIndex(this.indexPath);
|
||||||
|
|
||||||
this.initialized = true;
|
this.initialized = true;
|
||||||
console.log(`FAISS index initialized at ${this.indexPath}`);
|
logger.info('FAISS index initialized', { indexPath: this.indexPath });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add an enrollment embedding to the index.
|
* Add an enrollment embedding to the index.
|
||||||
*/
|
*/
|
||||||
async add(enrollmentId: string, embedding: number[]): Promise<void> {
|
async add(enrollmentId: string, embedding: number[]): Promise<void> {
|
||||||
@@ -546,7 +595,7 @@ export class FAISSIndex {
|
|||||||
// TODO: Add to FAISS index
|
// TODO: Add to FAISS index
|
||||||
// this.index.add([embedding]);
|
// this.index.add([embedding]);
|
||||||
// Store mapping: enrollmentId -> index position
|
// Store mapping: enrollmentId -> index position
|
||||||
console.log(`Added enrollment ${enrollmentId} to FAISS index`);
|
logger.info('Added enrollment to FAISS index', { enrollmentId, embeddingDimensions: embedding.length });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -556,7 +605,7 @@ export class FAISSIndex {
|
|||||||
await this.initialize();
|
await this.initialize();
|
||||||
|
|
||||||
// TODO: Remove from FAISS index
|
// TODO: Remove from FAISS index
|
||||||
console.log(`Removed enrollment ${enrollmentId} from FAISS index`);
|
logger.info('Removed enrollment from FAISS index', { enrollmentId });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -576,19 +625,25 @@ export class FAISSIndex {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save the index to disk.
|
* Save the index to disk.
|
||||||
*/
|
*/
|
||||||
async save(): Promise<void> {
|
async save(): Promise<void> {
|
||||||
await this.initialize();
|
await this.initialize();
|
||||||
// TODO: Write FAISS index to disk
|
// TODO: Write FAISS index to disk
|
||||||
console.log(`FAISS index saved to ${this.indexPath}`);
|
logger.info('FAISS index saved', { indexPath: this.indexPath });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Export singleton instances
|
// Export classes only - use dependency injection for instantiation
|
||||||
|
// Deprecated singleton exports kept for backward compatibility only
|
||||||
|
/** @deprecated Use `new AudioPreprocessor()` instead */
|
||||||
export const audioPreprocessor = new AudioPreprocessor();
|
export const audioPreprocessor = new AudioPreprocessor();
|
||||||
|
/** @deprecated Use `new VoiceEnrollmentService()` instead */
|
||||||
export const voiceEnrollmentService = new VoiceEnrollmentService();
|
export const voiceEnrollmentService = new VoiceEnrollmentService();
|
||||||
|
/** @deprecated Use `new AnalysisService()` instead */
|
||||||
export const analysisService = new AnalysisService();
|
export const analysisService = new AnalysisService();
|
||||||
|
/** @deprecated Use `new BatchAnalysisService()` instead */
|
||||||
export const batchAnalysisService = new BatchAnalysisService();
|
export const batchAnalysisService = new BatchAnalysisService();
|
||||||
|
/** @deprecated Use `new EmbeddingService()` instead */
|
||||||
export const embeddingService = new EmbeddingService();
|
export const embeddingService = new EmbeddingService();
|
||||||
|
|||||||
83
shieldai-workflow.md
Normal file
83
shieldai-workflow.md
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
# ShieldAI Code Review Workflow
|
||||||
|
|
||||||
|
## Current State (as of May 2, 2026)
|
||||||
|
|
||||||
|
### PR Backlog Status
|
||||||
|
- **Open PRs**: 0 (pending commits pushed to master)
|
||||||
|
- **Pending commits**: 1 commit pushed (FRE-4604) — remaining 6 were previously pushed
|
||||||
|
- **Last review cycle**: FRE-4500, FRE-4499, FRE-4612 (security findings — all done)
|
||||||
|
- **Branch protection**: Configured (see `branch-protection-rules.yaml`)
|
||||||
|
- **PR template**: Configured (`.gitea/pull_request_templates/default.md`)
|
||||||
|
|
||||||
|
### Resolved Bottlenecks
|
||||||
|
1. ✅ PR-based workflow established with PR template
|
||||||
|
2. ✅ Branch protection rules documented and configured
|
||||||
|
3. ✅ Code review checklist integrated into PR template
|
||||||
|
4. ✅ Security review findings integrated (FRE-4499, FRE-4500, FRE-4612 all done)
|
||||||
|
|
||||||
|
## PR Process
|
||||||
|
|
||||||
|
1. **Feature branch creation** from `gt/master`
|
||||||
|
2. **Development commits** with conventional commit format (include issue ID: `FRE-XXXX: description`)
|
||||||
|
3. **PR creation** against `gt/master`
|
||||||
|
4. **Required reviews**:
|
||||||
|
- Code Reviewer — all PRs
|
||||||
|
- Security Reviewer — for security-sensitive changes
|
||||||
|
5. **CI checks** pass (lint, typecheck, test)
|
||||||
|
6. **Merge** via squash or rebase
|
||||||
|
|
||||||
|
### Code Review Checklist
|
||||||
|
|
||||||
|
- [ ] Security impact assessment
|
||||||
|
- [ ] Test coverage verification
|
||||||
|
- [ ] Type checking (TypeScript)
|
||||||
|
- [ ] Linting compliance
|
||||||
|
- [ ] Documentation updates
|
||||||
|
- [ ] Breaking changes documented
|
||||||
|
- [ ] Backward compatibility verified
|
||||||
|
|
||||||
|
### Branch Protection Rules
|
||||||
|
|
||||||
|
See `branch-protection-rules.yaml` for the full configuration. Summary:
|
||||||
|
|
||||||
|
- **Protected branch**: `gt/master`
|
||||||
|
- **Required reviews**: 1 approved review before merge
|
||||||
|
- **Required status checks**: lint, typecheck, test
|
||||||
|
- **Enforce admins**: false (admins can bypass during emergencies)
|
||||||
|
- **Allow force pushes**: true (for recovery scenarios)
|
||||||
|
|
||||||
|
## Review Assignment Policy
|
||||||
|
|
||||||
|
| Change Type | Required Reviewers |
|
||||||
|
|-------------|-------------------|
|
||||||
|
| General code | Code Reviewer |
|
||||||
|
| Security-critical | Code Reviewer + Security Reviewer |
|
||||||
|
| API contracts | Code Reviewer + CTO |
|
||||||
|
| Database schema | Code Reviewer + Senior Engineer |
|
||||||
|
|
||||||
|
## Review Pipeline
|
||||||
|
|
||||||
|
```
|
||||||
|
Engineer implements → marks in_review → Security Reviewer reviews → Code Reviewer reviews → Done
|
||||||
|
```
|
||||||
|
|
||||||
|
## Metrics to Track
|
||||||
|
|
||||||
|
- PR cycle time (creation to merge)
|
||||||
|
- Review turnaround time
|
||||||
|
- PR size (lines changed)
|
||||||
|
- Review comments per PR
|
||||||
|
- Merge conflict frequency
|
||||||
|
|
||||||
|
## Contribution Guidelines
|
||||||
|
|
||||||
|
1. Always create a feature branch from `gt/master`
|
||||||
|
2. Use conventional commit format: `type(scope): description (FRE-XXXX)`
|
||||||
|
3. Include tests for new functionality
|
||||||
|
4. Update documentation for API changes
|
||||||
|
5. Run lint and typecheck before pushing
|
||||||
|
6. Create PR with filled template before requesting review
|
||||||
|
7. Address all review comments before merge
|
||||||
|
|
||||||
|
---
|
||||||
|
*Updated from FRE-4556 audit, implemented in FRE-4661*
|
||||||
Reference in New Issue
Block a user