FRE-4533: Merge apps/{api,web,mobile} and shared-db into ShieldAI repo

Merge FrenoCorp apps into ShieldAI packages/:
- packages/api: merged routes (notifications), middleware (auth, rate-limit, error, logging), config, services (darkwatch, spamshield, voiceprint), tests
- packages/web: new SolidJS web app stub
- packages/mobile: new SolidJS mobile app stub
- packages/shared-db: new Prisma DB package (separate from existing packages/db)
- pnpm-workspace.yaml: restored (apps/* removed, already covered by packages/*)

Next: reconcile packages/shared-db with packages/db, and fix server.ts correlationRoutes import
This commit is contained in:
2026-05-02 10:19:11 -04:00
parent 1197fe48f7
commit e704a9074a
36 changed files with 0 additions and 978 deletions

View File

@@ -0,0 +1,174 @@
import { prisma, AlertType, AlertSeverity } from '@shieldsai/shared-db';
import {
NotificationService,
NotificationPriority,
loadNotificationConfig,
} from '@shieldsai/shared-notifications';
const ALERT_DEDUP_WINDOW_MS = 24 * 60 * 60 * 1000;
export class AlertPipeline {
private notificationService: NotificationService;
constructor() {
this.notificationService = new NotificationService(loadNotificationConfig());
}
async processNewExposures(exposureIds: string[]) {
const exposures = await prisma.exposure.findMany({
where: { id: { in: exposureIds }, isFirstTime: true },
include: {
subscription: {
select: {
id: true,
userId: true,
tier: true,
},
},
watchlistItem: true,
},
});
const alertsCreated: Awaited<ReturnType<typeof prisma.alert.create>>[] = [];
for (const exposure of exposures) {
const dedupKey = `exposure:${exposure.subscriptionId}:${exposure.source}:${exposure.identifierHash}`;
const recentAlert = await prisma.alert.findFirst({
where: {
subscriptionId: exposure.subscriptionId,
type: AlertType.exposure_detected,
createdAt: {
gte: new Date(Date.now() - ALERT_DEDUP_WINDOW_MS),
},
},
orderBy: { createdAt: 'desc' },
});
if (recentAlert) {
continue;
}
const alert = await prisma.alert.create({
data: {
subscriptionId: exposure.subscriptionId,
userId: exposure.subscription.userId,
exposureId: exposure.id,
type: AlertType.exposure_detected,
title: this.buildTitle(exposure),
message: this.buildMessage(exposure),
severity: this.mapSeverity(exposure.severity),
channel: this.getChannelsForTier(exposure.subscription.tier),
},
});
alertsCreated.push(alert);
await this.dispatchNotification(alert, exposure);
}
return alertsCreated;
}
async dispatchScanCompleteAlert(
subscriptionId: string,
userId: string,
exposuresFound: number
) {
const subscription = await prisma.subscription.findUnique({
where: { id: subscriptionId },
select: { tier: true },
});
if (!subscription) return;
const alert = await prisma.alert.create({
data: {
subscriptionId,
userId,
type: AlertType.scan_complete,
title: 'DarkWatch Scan Complete',
message: `Scan found ${exposuresFound} new exposure${exposuresFound === 1 ? '' : 's'}.`,
severity: exposuresFound > 0 ? 'warning' : 'info',
channel: this.getChannelsForTier(subscription.tier),
},
});
await this.dispatchNotification(alert, {
source: 'hibp',
severity: 'info',
identifier: '',
dataType: 'email',
} as any);
return alert;
}
private async dispatchNotification(
alert: {
userId: string;
channel: string[];
title: string;
message: string;
severity: AlertSeverity;
},
exposure: { source: string; severity: string; identifier: string; dataType: string }
) {
try {
if (!this.notificationService.isFullyConfigured()) return;
await this.notificationService.sendMultiChannelNotification(
{
userId: alert.userId,
},
alert.channel as any,
alert.title,
`<p>${alert.message}</p>
<p><strong>Source:</strong> ${exposure.source}</p>
<p><strong>Severity:</strong> ${exposure.severity}</p>
<p><strong>Type:</strong> ${exposure.dataType}</p>`,
alert.severity === 'critical'
? NotificationPriority.HIGH
: NotificationPriority.NORMAL
);
} catch (error) {
console.error('[AlertPipeline] Notification dispatch error:', error);
}
}
private buildTitle(exposure: {
source: string;
dataType: string;
severity: string;
}): string {
return `${exposure.severity.toUpperCase()}: ${exposure.dataType} exposure on ${exposure.source}`;
}
private buildMessage(exposure: {
identifier: string;
source: string;
severity: string;
dataType: string;
}): string {
const masked = exposure.identifier.includes('@')
? exposure.identifier.replace(/(?<=.{2}).*(?=@)/, '***')
: exposure.identifier.slice(0, 3) + '***';
return `Your ${exposure.dataType} (${masked}) was found in a ${exposure.source} breach with ${exposure.severity} severity.`;
}
private mapSeverity(severity: string): AlertSeverity {
return severity as AlertSeverity;
}
private getChannelsForTier(tier: string): string[] {
const channelMap: Record<string, string[]> = {
basic: ['email'],
plus: ['email', 'push'],
premium: ['email', 'push', 'sms'],
};
return channelMap[tier] || ['email'];
}
}
export const alertPipeline = new AlertPipeline();

View File

@@ -0,0 +1,5 @@
export { watchlistService } from './watchlist.service';
export { scanService } from './scan.service';
export { schedulerService } from './scheduler.service';
export { webhookService } from './webhook.service';
export { alertPipeline } from './alert.pipeline';

View File

@@ -0,0 +1,220 @@
import { prisma, ExposureSource, ExposureSeverity, WatchlistType } from '@shieldsai/shared-db';
import { createHash } from 'crypto';
function hashIdentifier(identifier: string): string {
return createHash('sha256').update(identifier.toLowerCase().trim()).digest('hex');
}
function determineSeverity(
source: ExposureSource,
dataType: WatchlistType
): ExposureSeverity {
const criticalSources = [ExposureSource.darkWebForum, ExposureSource.honeypot];
const warningSources = [ExposureSource.hibp, ExposureSource.shodan];
const criticalTypes = [WatchlistType.ssn];
if (criticalTypes.includes(dataType)) return ExposureSeverity.critical;
if (criticalSources.includes(source)) return ExposureSeverity.critical;
if (warningSources.includes(source)) return ExposureSeverity.warning;
return ExposureSeverity.info;
}
export class ScanService {
async checkHIBP(email: string): Promise<{ exposed: boolean; sources: string[] }> {
try {
const response = await fetch(
`https://hibp.com/api/v2/${encodeURIComponent(email)}`,
{
headers: {
'hibp-api-key': process.env.HIBP_API_KEY || '',
Accept: 'application/json',
},
signal: AbortSignal.timeout(15000),
}
);
if (response.status === 404) {
return { exposed: false, sources: [] };
}
if (!response.ok) {
console.error(`[ScanService:HIBP] Status ${response.status} for ${email}`);
return { exposed: false, sources: [] };
}
const data = await response.json();
const sources = Array.isArray(data)
? data.map((p: { Name: string }) => p.Name)
: [];
return { exposed: sources.length > 0, sources };
} catch (error) {
console.error('[ScanService:HIBP] Error:', error);
return { exposed: false, sources: [] };
}
}
async checkShodan(domain: string): Promise<{ exposed: boolean; ports: string[]; ips: string[] }> {
try {
const response = await fetch(
`https://api.shodan.io/shodan/host/${encodeURIComponent(domain)}`,
{
headers: {
Authorization: `Bearer ${process.env.SHODAN_API_KEY || ''}`,
},
signal: AbortSignal.timeout(15000),
}
);
if (response.status === 404) {
return { exposed: false, ports: [], ips: [] };
}
if (!response.ok) {
console.error(`[ScanService:Shodan] Status ${response.status} for ${domain}`);
return { exposed: false, ports: [], ips: [] };
}
const data = await response.json();
return {
exposed: !!data.ip_str,
ports: data.ports?.map(String) || [],
ips: [data.ip_str || ''],
};
} catch (error) {
console.error('[ScanService:Shodan] Error:', error);
return { exposed: false, ports: [], ips: [] };
}
}
async processSubscriptionScan(
subscriptionId: string,
watchlistItems: Awaited<ReturnType<ScanService['getWatchlistItems']>>
): Promise<{ exposuresCreated: number; exposuresUpdated: number }> {
let exposuresCreated = 0;
let exposuresUpdated = 0;
for (const item of watchlistItems) {
const identifier = item.value;
const identifierHash = hashIdentifier(identifier);
switch (item.type) {
case WatchlistType.email: {
const hibpResult = await this.checkHIBP(identifier);
if (hibpResult.exposed) {
for (const source of hibpResult.sources) {
const existing = await prisma.exposure.findFirst({
where: {
subscriptionId,
source: ExposureSource.hibp,
identifierHash,
metadata: { path: ['dbName'], equals: source },
},
});
if (existing) {
await prisma.exposure.update({
where: { id: existing.id },
data: { detectedAt: new Date() },
});
exposuresUpdated++;
} else {
await prisma.exposure.create({
data: {
subscriptionId,
watchlistItemId: item.id,
source: ExposureSource.hibp,
dataType: item.type,
identifier,
identifierHash,
severity: determineSeverity(ExposureSource.hibp, item.type),
isFirstTime: true,
metadata: { dbName: source },
detectedAt: new Date(),
},
});
exposuresCreated++;
}
}
}
break;
}
case WatchlistType.domain: {
const shodanResult = await this.checkShodan(identifier);
if (shodanResult.exposed) {
const existing = await prisma.exposure.findFirst({
where: {
subscriptionId,
source: ExposureSource.shodan,
identifierHash,
},
});
if (existing) {
await prisma.exposure.update({
where: { id: existing.id },
data: {
detectedAt: new Date(),
metadata: { ports: shodanResult.ports, ips: shodanResult.ips },
},
});
exposuresUpdated++;
} else {
await prisma.exposure.create({
data: {
subscriptionId,
watchlistItemId: item.id,
source: ExposureSource.shodan,
dataType: item.type,
identifier,
identifierHash,
severity: determineSeverity(ExposureSource.shodan, item.type),
isFirstTime: true,
metadata: { ports: shodanResult.ports, ips: shodanResult.ips },
detectedAt: new Date(),
},
});
exposuresCreated++;
}
}
break;
}
default: {
const existing = await prisma.exposure.findFirst({
where: { subscriptionId, watchlistItemId: item.id, identifierHash },
});
if (!existing) {
await prisma.exposure.create({
data: {
subscriptionId,
watchlistItemId: item.id,
source: ExposureSource.darkWebForum,
dataType: item.type,
identifier,
identifierHash,
severity: determineSeverity(ExposureSource.darkWebForum, item.type),
isFirstTime: true,
detectedAt: new Date(),
},
});
exposuresCreated++;
}
break;
}
}
}
return { exposuresCreated, exposuresUpdated };
}
async getWatchlistItems(subscriptionId: string) {
return prisma.watchlistItem.findMany({
where: { subscriptionId, isActive: true },
});
}
}
export const scanService = new ScanService();

View File

@@ -0,0 +1,155 @@
import { prisma, SubscriptionTier, SubscriptionStatus } from '@shieldsai/shared-db';
import { tierConfig } from '@shieldsai/shared-billing';
import { darkwatchScanQueue } from '@shieldsai/jobs';
import { randomUUID } from 'crypto';
const CRON_EXPRESSIONS = {
daily: '0 0 * * *',
hourly: '0 * * * *',
realtime: null,
};
export class SchedulerService {
async scheduleSubscriptionScans() {
const activeSubscriptions = await prisma.subscription.findMany({
where: {
tier: { in: [SubscriptionTier.basic, SubscriptionTier.plus, SubscriptionTier.premium] },
status: SubscriptionStatus.active,
},
select: {
id: true,
tier: true,
userId: true,
},
});
const jobsEnqueued = [];
for (const subscription of activeSubscriptions) {
const frequency = tierConfig[subscription.tier].features.darkWebScanFrequency;
const cron = CRON_EXPRESSIONS[frequency];
if (!cron) {
continue;
}
const jobKey = `scheduled-scan:${subscription.id}`;
try {
await darkwatchScanQueue.add(
'scheduled-scan',
{
subscriptionId: subscription.id,
tier: subscription.tier,
scanType: 'scheduled',
},
{
jobId: jobKey,
repeat: {
every: frequency === 'daily'
? 24 * 60 * 60 * 1000
: 60 * 60 * 1000,
},
priority: subscription.tier === SubscriptionTier.premium ? 1 : 3,
}
);
jobsEnqueued.push({
subscriptionId: subscription.id,
tier: subscription.tier,
frequency,
});
} catch (error) {
if ((error as Error).message?.includes('Duplicate')) {
continue;
}
console.error(
`[SchedulerService] Failed to schedule scan for ${subscription.id}:`,
error
);
}
}
return jobsEnqueued;
}
async enqueueOnDemandScan(subscriptionId: string) {
const subscription = await prisma.subscription.findUnique({
where: { id: subscriptionId },
select: { id: true, tier: true },
});
if (!subscription) {
throw new Error(`Subscription ${subscriptionId} not found`);
}
return darkwatchScanQueue.add(
'on-demand-scan',
{
subscriptionId,
tier: subscription.tier,
scanType: 'on-demand',
},
{
priority: 1,
jobId: `on-demand-scan:${subscriptionId}:${randomUUID()}`,
}
);
}
async enqueueRealtimeTrigger(subscriptionId: string, sourceData: Record<string, unknown>) {
const subscription = await prisma.subscription.findUnique({
where: { id: subscriptionId },
select: { id: true, tier: true },
});
if (!subscription || subscription.tier !== SubscriptionTier.premium) {
throw new Error('Realtime triggers require Premium tier');
}
return darkwatchScanQueue.add(
'realtime-trigger',
{
subscriptionId,
tier: subscription.tier,
scanType: 'realtime',
sourceData,
},
{
priority: 0,
jobId: `realtime-trigger:${subscriptionId}:${randomUUID()}`,
}
);
}
async rescheduleAll() {
const repeatableJobs = await darkwatchScanQueue.getRepeatableJobs();
for (const job of repeatableJobs) {
await darkwatchScanQueue.removeRepeatableByKey(job.key);
}
return this.scheduleSubscriptionScans();
}
async getScanSchedule(subscriptionId: string) {
const subscription = await prisma.subscription.findUnique({
where: { id: subscriptionId },
select: { tier: true },
});
if (!subscription) return null;
const frequency = tierConfig[subscription.tier].features.darkWebScanFrequency;
return {
subscriptionId,
tier: subscription.tier,
frequency,
cron: CRON_EXPRESSIONS[frequency],
nextRun: frequency === 'realtime' ? 'event-driven' : CRON_EXPRESSIONS[frequency],
};
}
}
export const schedulerService = new SchedulerService();

View File

@@ -0,0 +1,97 @@
import { prisma, WatchlistType } from '@shieldsai/shared-db';
import { createHash } from 'crypto';
export function normalizeValue(type: WatchlistType, value: string): string {
const trimmed = value.trim().toLowerCase();
switch (type) {
case WatchlistType.email:
return trimmed.replace(/\s+/g, '');
case WatchlistType.phoneNumber:
return trimmed.replace(/[\s\-\(\)]/g, '');
case WatchlistType.ssn:
return trimmed.replace(/-/g, '');
case WatchlistType.address:
return trimmed;
case WatchlistType.domain:
return trimmed.replace(/^https?:\/\//, '').replace(/\/.*$/, '');
default:
return trimmed;
}
}
export function hashValue(value: string): string {
return createHash('sha256').update(value).digest('hex');
}
export class WatchlistService {
async addItem(
subscriptionId: string,
type: WatchlistType,
value: string,
maxItems: number
) {
const normalized = normalizeValue(type, value);
const itemHash = hashValue(normalized);
const currentCount = await prisma.watchlistItem.count({
where: { subscriptionId, isActive: true },
});
if (currentCount >= maxItems) {
throw new Error(
`Watchlist limit reached (${maxItems} items). Upgrade your plan to add more.`
);
}
const existing = await prisma.watchlistItem.findFirst({
where: { subscriptionId, type, hash: itemHash },
});
if (existing) {
if (!existing.isActive) {
return prisma.watchlistItem.update({
where: { id: existing.id },
data: { isActive: true },
});
}
return existing;
}
return prisma.watchlistItem.create({
data: {
subscriptionId,
type,
value: normalized,
hash: itemHash,
},
});
}
async getItems(subscriptionId: string) {
return prisma.watchlistItem.findMany({
where: { subscriptionId, isActive: true },
orderBy: { createdAt: 'desc' },
});
}
async removeItem(id: string, subscriptionId: string) {
return prisma.watchlistItem.update({
where: { id },
data: { isActive: false },
});
}
async getActiveItemsForScan(subscriptionId: string) {
return prisma.watchlistItem.findMany({
where: { subscriptionId, isActive: true },
});
}
async getItemCount(subscriptionId: string) {
return prisma.watchlistItem.count({
where: { subscriptionId, isActive: true },
});
}
}
export const watchlistService = new WatchlistService();

View File

@@ -0,0 +1,226 @@
import { prisma, ExposureSource, ExposureSeverity, WatchlistType, AlertType, AlertSeverity } from '@shieldsai/shared-db';
import { createHash } from 'crypto';
import { mixpanelService, EventType } from '@shieldsai/shared-analytics';
function hashIdentifier(identifier: string): string {
return createHash('sha256').update(identifier.toLowerCase().trim()).digest('hex');
}
function determineSeverity(
source: ExposureSource,
dataType: WatchlistType
): ExposureSeverity {
const criticalSources = [ExposureSource.darkWebForum, ExposureSource.honeypot];
const warningSources = [ExposureSource.hibp, ExposureSource.shodan];
const criticalTypes = [WatchlistType.ssn];
if (criticalTypes.includes(dataType)) return ExposureSeverity.critical;
if (criticalSources.includes(source)) return ExposureSeverity.critical;
if (warningSources.includes(source)) return ExposureSeverity.warning;
return ExposureSeverity.info;
}
export interface WebhookPayload {
source: string;
identifier: string;
identifierType: string;
metadata?: Record<string, unknown>;
timestamp?: string;
}
export class WebhookService {
async processExternalWebhook(payload: WebhookPayload): Promise<{
exposuresCreated: number;
alertsCreated: number;
}> {
const source = this.mapSource(payload.source);
const dataType = this.mapDataType(payload.identifierType);
const identifier = payload.identifier.toLowerCase().trim();
const identifierHash = hashIdentifier(identifier);
const severity = determineSeverity(source, dataType);
const matchingItems = await prisma.watchlistItem.findMany({
where: {
isActive: true,
OR: [
{ hash: identifierHash, type: dataType },
{ value: identifier, type: dataType },
],
},
include: {
subscription: {
select: {
id: true,
tier: true,
userId: true,
},
},
},
});
let exposuresCreated = 0;
let alertsCreated = 0;
for (const item of matchingItems) {
const existing = await prisma.exposure.findFirst({
where: {
subscriptionId: item.subscriptionId,
source,
identifierHash,
},
});
if (existing) {
await prisma.exposure.update({
where: { id: existing.id },
data: { detectedAt: new Date() },
});
continue;
}
const exposure = await prisma.exposure.create({
data: {
subscriptionId: item.subscriptionId,
watchlistItemId: item.id,
source,
dataType,
identifier,
identifierHash,
severity,
isFirstTime: true,
metadata: payload.metadata || {},
detectedAt: new Date(),
},
});
exposuresCreated++;
const alertChannels = this.getAlertChannelsForTier(item.subscription.tier);
await prisma.alert.create({
data: {
subscriptionId: item.subscriptionId,
userId: item.subscription.userId,
exposureId: exposure.id,
type: AlertType.exposure_detected,
title: `New Exposure Detected: ${this.getSourceLabel(source)}`,
message: this.buildAlertMessage(identifier, source, severity),
severity: this.mapAlertSeverity(severity),
channel: alertChannels,
},
});
alertsCreated++;
await mixpanelService.track(EventType.EXPOSURE_DETECTED, {
userId: item.subscription.userId,
exposureType: dataType,
severity,
source,
subscriptionTier: item.subscription.tier,
});
}
return { exposuresCreated, alertsCreated };
}
async verifyWebhookSignature(
body: string,
signature: string,
timestamp: string
): Promise<boolean> {
const webhookSecret = process.env.DARKWATCH_WEBHOOK_SECRET;
if (!webhookSecret) {
console.warn('[WebhookService] DARKWATCH_WEBHOOK_SECRET not set — signature verification skipped');
return false;
}
const expected = createHash('sha256')
.update(`${timestamp}:${body}`)
.digest('hex');
return expected === signature;
}
private mapSource(source: string): ExposureSource {
const sourceMap: Record<string, ExposureSource> = {
hibp: ExposureSource.hibp,
'haveibeenpwned': ExposureSource.hibp,
securitytrails: ExposureSource.securityTrails,
censys: ExposureSource.censys,
'darkweb-forum': ExposureSource.darkWebForum,
'darkweb': ExposureSource.darkWebForum,
shodan: ExposureSource.shodan,
honeypot: ExposureSource.honeypot,
};
const normalized = source.toLowerCase().replace(/\s+/g, '');
const mapped = sourceMap[normalized];
if (!mapped) {
console.warn(`[WebhookService] Unknown source "${source}", falling back to darkWebForum`);
}
return mapped || ExposureSource.darkWebForum;
}
private mapDataType(type: string): WatchlistType {
const typeMap: Record<string, WatchlistType> = {
email: WatchlistType.email,
phone: WatchlistType.phoneNumber,
phonenumber: WatchlistType.phoneNumber,
ssn: WatchlistType.ssn,
address: WatchlistType.address,
domain: WatchlistType.domain,
};
const normalized = type.toLowerCase().trim();
return typeMap[normalized] || WatchlistType.email;
}
private getAlertChannelsForTier(tier: string): string[] {
const channelMap: Record<string, string[]> = {
basic: ['email'],
plus: ['email', 'push'],
premium: ['email', 'push', 'sms'],
};
return channelMap[tier] || ['email'];
}
private mapAlertSeverity(severity: ExposureSeverity): AlertSeverity {
return severity as AlertSeverity;
}
private getSourceLabel(source: ExposureSource): string {
const labels: Record<ExposureSource, string> = {
[ExposureSource.hibp]: 'Have I Been Pwned',
[ExposureSource.securityTrails]: 'SecurityTrails',
[ExposureSource.censys]: 'Censys',
[ExposureSource.darkWebForum]: 'Dark Web Forum',
[ExposureSource.shodan]: 'Shodan',
[ExposureSource.honeypot]: 'Honeypot',
};
return labels[source] || source;
}
private buildAlertMessage(
identifier: string,
source: ExposureSource,
severity: ExposureSeverity
): string {
const masked = this.maskIdentifier(identifier);
return `${severity.toUpperCase()}: "${masked}" found in ${this.getSourceLabel(source)}.`;
}
private maskIdentifier(identifier: string): string {
if (identifier.includes('@')) {
const [user, domain] = identifier.split('@');
const maskedUser = user.slice(0, 2) + '***' + user.slice(-1);
return `${maskedUser}@${domain}`;
}
if (identifier.length > 8) {
return identifier.slice(0, 3) + '***' + identifier.slice(-2);
}
return identifier;
}
}
export const webhookService = new WebhookService();