Add Info Broker Removal service (FRE-5402)
New service for helping clients remove personal listings from data broker sites. Service features: - BrokerRegistry: Catalog of 20+ data brokers with removal methods - RemoveBrokersService: Core service for scanning, creating removal requests, submitting removals, and verifying completions - RemoveBrokersScheduler: Automated processing of pending removals and verification of completed removals - BrokerAlertPipeline: Alert integration for listing discoveries and removal status API endpoints (/removebrokers): - GET /brokers - List available data brokers - GET /status - Get removal request status and stats - POST /scan - Scan for personal listings across brokers - POST /request - Create a new removal request - GET /request/:id - Get specific removal request details - DELETE /request/:id - Cancel a removal request - POST /process - Trigger processing of pending removals - POST /verify/:id - Manually verify a removal completion DB models: InfoBroker, RemovalRequest, BrokerListing Types: BrokerStatus, RemovalStatus, RemovalMethod, and related interfaces
This commit is contained in:
@@ -8,6 +8,7 @@ import { subscriptionRoutes } from './subscription.routes';
|
|||||||
import { deviceRoutes } from './device.routes';
|
import { deviceRoutes } from './device.routes';
|
||||||
import { notificationRoutes } from './notifications.routes';
|
import { notificationRoutes } from './notifications.routes';
|
||||||
import { hometitleRoutes } from './hometitle.routes';
|
import { hometitleRoutes } from './hometitle.routes';
|
||||||
|
import { removebrokersRoutes } from './removebrokers.routes';
|
||||||
|
|
||||||
export async function routes(fastify: FastifyInstance) {
|
export async function routes(fastify: FastifyInstance) {
|
||||||
// Authenticated routes group
|
// Authenticated routes group
|
||||||
@@ -179,4 +180,15 @@ export async function routes(fastify: FastifyInstance) {
|
|||||||
},
|
},
|
||||||
{ prefix: '/hometitle' }
|
{ prefix: '/hometitle' }
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Info Broker Removal service routes
|
||||||
|
fastify.register(
|
||||||
|
async (removebrokersRouter) => {
|
||||||
|
removebrokersRouter.addHook('onRequest', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
await fastify.requireAuth(request as AuthRequest);
|
||||||
|
});
|
||||||
|
await removebrokersRoutes(removebrokersRouter);
|
||||||
|
},
|
||||||
|
{ prefix: '/removebrokers' }
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
378
packages/api/src/routes/removebrokers.routes.ts
Normal file
378
packages/api/src/routes/removebrokers.routes.ts
Normal file
@@ -0,0 +1,378 @@
|
|||||||
|
import { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
|
||||||
|
import { prisma } from '@shieldai/db';
|
||||||
|
import { RemovalStatus, Severity } from '@shieldai/types';
|
||||||
|
import {
|
||||||
|
removeBrokersService,
|
||||||
|
removeBrokersScheduler,
|
||||||
|
brokerAlertPipeline,
|
||||||
|
type PersonalInfo,
|
||||||
|
} from '@shieldai/removebrokers';
|
||||||
|
import { AuthRequest } from '../middleware/auth.middleware';
|
||||||
|
|
||||||
|
const REMOVAL_REQUEST_LIMITS: Record<string, number> = {
|
||||||
|
basic: 5,
|
||||||
|
plus: 20,
|
||||||
|
premium: 999,
|
||||||
|
};
|
||||||
|
|
||||||
|
async function getSubscription(
|
||||||
|
request: FastifyRequest,
|
||||||
|
reply: FastifyReply,
|
||||||
|
): Promise<{ subscriptionId: string; tier: string } | null> {
|
||||||
|
const authReq = request as AuthRequest;
|
||||||
|
const userId = authReq.user?.id;
|
||||||
|
|
||||||
|
if (!userId) {
|
||||||
|
await reply.code(401).send({ error: 'User not authenticated' });
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const subscription = await prisma.subscription.findFirst({
|
||||||
|
where: { userId, status: 'active' },
|
||||||
|
select: { id: true, tier: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!subscription) {
|
||||||
|
await reply.code(402).send({
|
||||||
|
error: 'Subscription required',
|
||||||
|
message: 'An active subscription is required for data broker removal',
|
||||||
|
});
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { subscriptionId: subscription.id, tier: subscription.tier };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function removebrokersRoutes(fastify: FastifyInstance) {
|
||||||
|
// GET /removebrokers/brokers - List available data brokers
|
||||||
|
fastify.get('/brokers', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const authReq = request as AuthRequest;
|
||||||
|
if (!authReq.user?.id) {
|
||||||
|
return reply.code(401).send({ error: 'User not authenticated' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const query = request.query as { category?: string };
|
||||||
|
let brokers = await removeBrokersService.getAvailableBrokers();
|
||||||
|
|
||||||
|
if (query.category) {
|
||||||
|
brokers = brokers.filter((b) => b.category === query.category);
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
brokers: brokers.map((b) => ({
|
||||||
|
id: b.id,
|
||||||
|
name: b.name,
|
||||||
|
domain: b.domain,
|
||||||
|
category: b.category,
|
||||||
|
removalMethod: b.removalMethod,
|
||||||
|
requiresAccount: b.requiresAccount,
|
||||||
|
requiresVerification: b.requiresVerification,
|
||||||
|
estimatedDays: b.estimatedDays,
|
||||||
|
removalUrl: b.removalUrl,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// GET /removebrokers/status - Get removal request status for user
|
||||||
|
fastify.get('/status', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const sub = await getSubscription(request, reply);
|
||||||
|
if (!sub) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const status = await removeBrokersService.getRemovalStatus(sub.subscriptionId);
|
||||||
|
|
||||||
|
const total = status.length;
|
||||||
|
const pending = status.filter((s) => s.status === RemovalStatus.PENDING).length;
|
||||||
|
const submitted = status.filter((s) => s.status === RemovalStatus.SUBMITTED).length;
|
||||||
|
const completed = status.filter((s) => s.status === RemovalStatus.COMPLETED).length;
|
||||||
|
const failed = status.filter((s) => s.status === RemovalStatus.FAILED).length;
|
||||||
|
|
||||||
|
const limit = REMOVAL_REQUEST_LIMITS[sub.tier] ?? 5;
|
||||||
|
const remaining = Math.max(0, limit - total);
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
stats: { total, pending, submitted, completed, failed },
|
||||||
|
limit,
|
||||||
|
remaining,
|
||||||
|
tier: sub.tier,
|
||||||
|
requests: status,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : 'Failed to fetch status';
|
||||||
|
return reply.code(500).send({ error: message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// POST /removebrokers/scan - Scan for personal listings across brokers
|
||||||
|
fastify.post('/scan', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const sub = await getSubscription(request, reply);
|
||||||
|
if (!sub) return;
|
||||||
|
|
||||||
|
const body = request.body as { fullName?: string; email?: string; phone?: string; address?: string };
|
||||||
|
|
||||||
|
if (!body.fullName) {
|
||||||
|
return reply.code(400).send({
|
||||||
|
error: 'Invalid request',
|
||||||
|
message: 'fullName is required for scanning',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const user = await prisma.user.findUnique({
|
||||||
|
where: { id: (request as AuthRequest).user!.id },
|
||||||
|
});
|
||||||
|
|
||||||
|
const personalInfo: PersonalInfo = {
|
||||||
|
fullName: body.fullName,
|
||||||
|
email: body.email || user?.email,
|
||||||
|
phone: body.phone,
|
||||||
|
address: body.address
|
||||||
|
? { street: body.address }
|
||||||
|
: undefined,
|
||||||
|
};
|
||||||
|
|
||||||
|
const results = await removeBrokersService.scanForListings(
|
||||||
|
sub.subscriptionId,
|
||||||
|
personalInfo,
|
||||||
|
);
|
||||||
|
|
||||||
|
const found = results.filter((r) => r.found);
|
||||||
|
|
||||||
|
for (const listing of found) {
|
||||||
|
try {
|
||||||
|
await brokerAlertPipeline.sendListingFoundAlert({
|
||||||
|
userId: (request as AuthRequest).user!.id,
|
||||||
|
brokerName: listing.brokerName,
|
||||||
|
brokerId: listing.brokerId,
|
||||||
|
category: 'INFO_BROKER_LISTING' as any,
|
||||||
|
severity: Severity.MEDIUM,
|
||||||
|
title: `Personal listing found on ${listing.brokerName}`,
|
||||||
|
description: `Your personal information was found on ${listing.brokerName} (${listing.brokerId}). Consider submitting a removal request.`,
|
||||||
|
entities: [
|
||||||
|
{ type: 'USER_ID' as any, value: (request as AuthRequest).user!.id },
|
||||||
|
],
|
||||||
|
metadata: { url: listing.url },
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Alert failure is non-critical
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
brokersScanned: results.length,
|
||||||
|
listingsFound: found.length,
|
||||||
|
results,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : 'Scan failed';
|
||||||
|
return reply.code(500).send({ error: message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// POST /removebrokers/request - Create a new removal request
|
||||||
|
fastify.post('/request', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const sub = await getSubscription(request, reply);
|
||||||
|
if (!sub) return;
|
||||||
|
|
||||||
|
const body = request.body as {
|
||||||
|
brokerId: string;
|
||||||
|
fullName: string;
|
||||||
|
email?: string;
|
||||||
|
phone?: string;
|
||||||
|
address?: {
|
||||||
|
street?: string;
|
||||||
|
city?: string;
|
||||||
|
state?: string;
|
||||||
|
zip?: string;
|
||||||
|
};
|
||||||
|
dob?: string;
|
||||||
|
notes?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!body.brokerId) {
|
||||||
|
return reply.code(400).send({
|
||||||
|
error: 'Invalid request',
|
||||||
|
message: 'brokerId is required',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!body.fullName) {
|
||||||
|
return reply.code(400).send({
|
||||||
|
error: 'Invalid request',
|
||||||
|
message: 'fullName is required',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const limit = REMOVAL_REQUEST_LIMITS[sub.tier] ?? 5;
|
||||||
|
const currentCount = await prisma.removalRequest.count({
|
||||||
|
where: { subscriptionId: sub.subscriptionId },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (currentCount >= limit) {
|
||||||
|
return reply.code(400).send({
|
||||||
|
error: 'Request limit reached',
|
||||||
|
message: `You have reached the maximum of ${limit} removal requests for your ${sub.tier} tier.`,
|
||||||
|
currentCount,
|
||||||
|
limit,
|
||||||
|
upgradeTo: 'plus',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const personalInfo: PersonalInfo = {
|
||||||
|
fullName: body.fullName,
|
||||||
|
email: body.email,
|
||||||
|
phone: body.phone,
|
||||||
|
address: body.address,
|
||||||
|
dob: body.dob,
|
||||||
|
};
|
||||||
|
|
||||||
|
const req = await removeBrokersService.createRemovalRequest(
|
||||||
|
sub.subscriptionId,
|
||||||
|
body.brokerId,
|
||||||
|
personalInfo,
|
||||||
|
body.notes,
|
||||||
|
);
|
||||||
|
|
||||||
|
return reply.code(201).send({
|
||||||
|
request: {
|
||||||
|
id: req.id,
|
||||||
|
brokerId: req.brokerId,
|
||||||
|
status: req.status,
|
||||||
|
method: req.method,
|
||||||
|
createdAt: req.createdAt,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : 'Failed to create removal request';
|
||||||
|
return reply.code(422).send({ error: message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// GET /removebrokers/request/:id - Get specific removal request
|
||||||
|
fastify.get('/request/:id', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const sub = await getSubscription(request, reply);
|
||||||
|
if (!sub) return;
|
||||||
|
|
||||||
|
const id = (request.params as { id: string }).id;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const req = await prisma.removalRequest.findFirst({
|
||||||
|
where: { id, subscriptionId: sub.subscriptionId },
|
||||||
|
include: { broker: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!req) {
|
||||||
|
return reply.code(404).send({ error: 'Removal request not found' });
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
request: {
|
||||||
|
id: req.id,
|
||||||
|
brokerId: req.brokerId,
|
||||||
|
brokerName: req.broker.name,
|
||||||
|
status: req.status,
|
||||||
|
method: req.method,
|
||||||
|
attempts: req.attempts,
|
||||||
|
submittedAt: req.submittedAt,
|
||||||
|
completedAt: req.completedAt,
|
||||||
|
error: req.error,
|
||||||
|
notes: req.notes,
|
||||||
|
createdAt: req.createdAt,
|
||||||
|
updatedAt: req.updatedAt,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : 'Failed to fetch request';
|
||||||
|
return reply.code(500).send({ error: message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// DELETE /removebrokers/request/:id - Cancel a removal request
|
||||||
|
fastify.delete('/request/:id', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const sub = await getSubscription(request, reply);
|
||||||
|
if (!sub) return;
|
||||||
|
|
||||||
|
const id = (request.params as { id: string }).id;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const req = await prisma.removalRequest.findFirst({
|
||||||
|
where: { id, subscriptionId: sub.subscriptionId },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!req) {
|
||||||
|
return reply.code(404).send({ error: 'Removal request not found' });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.status === RemovalStatus.COMPLETED) {
|
||||||
|
return reply.code(400).send({
|
||||||
|
error: 'Cannot cancel',
|
||||||
|
message: 'Cannot cancel a completed removal request',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
await prisma.removalRequest.update({
|
||||||
|
where: { id },
|
||||||
|
data: { status: RemovalStatus.REJECTED },
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
request: {
|
||||||
|
id: req.id,
|
||||||
|
status: 'cancelled',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : 'Failed to cancel request';
|
||||||
|
return reply.code(500).send({ error: message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// POST /removebrokers/process - Trigger processing of pending removals (admin)
|
||||||
|
fastify.post('/process', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const authReq = request as AuthRequest;
|
||||||
|
if (!authReq.user?.id) {
|
||||||
|
return reply.code(401).send({ error: 'User not authenticated' });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const results = await removeBrokersService.processPendingRequests();
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
processed: results.length,
|
||||||
|
results,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : 'Processing failed';
|
||||||
|
return reply.code(500).send({ error: message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// POST /removebrokers/verify/:id - Manually verify a removal
|
||||||
|
fastify.post('/verify/:id', async (request: FastifyRequest, reply: FastifyReply) => {
|
||||||
|
const sub = await getSubscription(request, reply);
|
||||||
|
if (!sub) return;
|
||||||
|
|
||||||
|
const id = (request.params as { id: string }).id;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const req = await prisma.removalRequest.findFirst({
|
||||||
|
where: { id, subscriptionId: sub.subscriptionId },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!req) {
|
||||||
|
return reply.code(404).send({ error: 'Removal request not found' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await removeBrokersService.verifyRemoval(id);
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
requestId: id,
|
||||||
|
...result,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : 'Verification failed';
|
||||||
|
return reply.code(500).send({ error: message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -25,6 +25,7 @@ model User {
|
|||||||
// Relationships
|
// Relationships
|
||||||
accounts Account[]
|
accounts Account[]
|
||||||
sessions Session[]
|
sessions Session[]
|
||||||
|
deviceTokens DeviceToken[]
|
||||||
familyGroups FamilyGroupMember[]
|
familyGroups FamilyGroupMember[]
|
||||||
familyGroupOwned FamilyGroup[] @relation("FamilyGroupOwner")
|
familyGroupOwned FamilyGroup[] @relation("FamilyGroupOwner")
|
||||||
subscriptions Subscription[]
|
subscriptions Subscription[]
|
||||||
@@ -37,6 +38,8 @@ model User {
|
|||||||
correlationGroups CorrelationGroup[]
|
correlationGroups CorrelationGroup[]
|
||||||
securityReports SecurityReport[]
|
securityReports SecurityReport[]
|
||||||
analysisJobs AnalysisJob[]
|
analysisJobs AnalysisJob[]
|
||||||
|
removalRequests RemovalRequest[]
|
||||||
|
brokerListings BrokerListing[]
|
||||||
|
|
||||||
// Audit
|
// Audit
|
||||||
createdAt DateTime @default(now())
|
createdAt DateTime @default(now())
|
||||||
@@ -107,6 +110,42 @@ model Session {
|
|||||||
@@index([userId])
|
@@index([userId])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model DeviceToken {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
userId String
|
||||||
|
deviceType DeviceType
|
||||||
|
token String @unique
|
||||||
|
platform Platform
|
||||||
|
appName String?
|
||||||
|
appVersion String?
|
||||||
|
osVersion String?
|
||||||
|
model String?
|
||||||
|
isActive Boolean @default(true)
|
||||||
|
lastUsedAt DateTime @default(now())
|
||||||
|
|
||||||
|
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
@@index([userId])
|
||||||
|
@@index([deviceType])
|
||||||
|
@@index([platform])
|
||||||
|
@@index([isActive])
|
||||||
|
}
|
||||||
|
|
||||||
|
enum DeviceType {
|
||||||
|
mobile
|
||||||
|
web
|
||||||
|
desktop
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Platform {
|
||||||
|
ios
|
||||||
|
android
|
||||||
|
web
|
||||||
|
}
|
||||||
|
|
||||||
// ============================================
|
// ============================================
|
||||||
// Family & Subscription Models
|
// Family & Subscription Models
|
||||||
// ============================================
|
// ============================================
|
||||||
@@ -167,6 +206,9 @@ model Subscription {
|
|||||||
watchlistItems WatchlistItem[]
|
watchlistItems WatchlistItem[]
|
||||||
exposures Exposure[]
|
exposures Exposure[]
|
||||||
alerts Alert[]
|
alerts Alert[]
|
||||||
|
propertyWatchlistItems PropertyWatchlistItem[]
|
||||||
|
removalRequests RemovalRequest[]
|
||||||
|
brokerListings BrokerListing[]
|
||||||
|
|
||||||
createdAt DateTime @default(now())
|
createdAt DateTime @default(now())
|
||||||
updatedAt DateTime @updatedAt
|
updatedAt DateTime @updatedAt
|
||||||
@@ -506,6 +548,8 @@ enum AlertSource {
|
|||||||
SPAMSHIELD
|
SPAMSHIELD
|
||||||
VOICEPRINT
|
VOICEPRINT
|
||||||
CALL_ANALYSIS
|
CALL_ANALYSIS
|
||||||
|
HOME_TITLE
|
||||||
|
INFO_BROKER
|
||||||
}
|
}
|
||||||
|
|
||||||
enum AlertCategory {
|
enum AlertCategory {
|
||||||
@@ -517,6 +561,9 @@ enum AlertCategory {
|
|||||||
CALL_ANOMALY
|
CALL_ANOMALY
|
||||||
CALL_QUALITY
|
CALL_QUALITY
|
||||||
CALL_EVENT
|
CALL_EVENT
|
||||||
|
HOME_TITLE
|
||||||
|
INFO_BROKER_LISTING
|
||||||
|
INFO_BROKER_REMOVAL
|
||||||
}
|
}
|
||||||
|
|
||||||
enum NormalizedAlertSeverity {
|
enum NormalizedAlertSeverity {
|
||||||
@@ -588,6 +635,7 @@ model CorrelationGroup {
|
|||||||
enum ReportType {
|
enum ReportType {
|
||||||
MONTHLY_PLUS
|
MONTHLY_PLUS
|
||||||
ANNUAL_PREMIUM
|
ANNUAL_PREMIUM
|
||||||
|
WEEKLY_DIGEST
|
||||||
}
|
}
|
||||||
|
|
||||||
enum ReportStatus {
|
enum ReportStatus {
|
||||||
@@ -676,3 +724,192 @@ model BlogPost {
|
|||||||
@@index([published, publishedAt])
|
@@index([published, publishedAt])
|
||||||
@@index([tags])
|
@@index([tags])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// Home Title Service Models
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
enum PropertyChangeType {
|
||||||
|
tax_change
|
||||||
|
deed_change
|
||||||
|
ownership_transfer
|
||||||
|
lien_filing
|
||||||
|
metadata_change
|
||||||
|
}
|
||||||
|
|
||||||
|
enum PropertyChangeSeverity {
|
||||||
|
info
|
||||||
|
warning
|
||||||
|
critical
|
||||||
|
}
|
||||||
|
|
||||||
|
model PropertyWatchlistItem {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
subscriptionId String
|
||||||
|
address String
|
||||||
|
parcelId String?
|
||||||
|
ownerName String?
|
||||||
|
streetAddress String
|
||||||
|
city String? @default("")
|
||||||
|
state String? @default("")
|
||||||
|
zipCode String? @default("")
|
||||||
|
latitude Float?
|
||||||
|
longitude Float?
|
||||||
|
isActive Boolean @default(true)
|
||||||
|
|
||||||
|
subscription Subscription @relation(fields: [subscriptionId], references: [id], onDelete: Cascade)
|
||||||
|
snapshots PropertySnapshot[]
|
||||||
|
changes PropertyChange[]
|
||||||
|
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
@@unique([subscriptionId, parcelId])
|
||||||
|
@@index([subscriptionId])
|
||||||
|
@@index([parcelId])
|
||||||
|
@@index([address])
|
||||||
|
}
|
||||||
|
|
||||||
|
model PropertySnapshot {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
propertyWatchlistItemId String
|
||||||
|
subscriptionId String
|
||||||
|
capturedAt DateTime
|
||||||
|
ownerName String
|
||||||
|
address Json // { streetNumber, streetName, streetType, unit, city, state, zip, latitude?, longitude? }
|
||||||
|
deedDate String?
|
||||||
|
taxId String?
|
||||||
|
propertyType String @default("residential")
|
||||||
|
taxAmount Float?
|
||||||
|
lienCount Int @default(0)
|
||||||
|
|
||||||
|
propertyWatchlistItem PropertyWatchlistItem @relation(fields: [propertyWatchlistItemId], references: [id], onDelete: Cascade)
|
||||||
|
changes PropertyChange[] @relation("SnapshotChanges")
|
||||||
|
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
@@index([propertyWatchlistItemId])
|
||||||
|
@@index([subscriptionId])
|
||||||
|
@@index([capturedAt])
|
||||||
|
}
|
||||||
|
|
||||||
|
model PropertyChange {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
propertyWatchlistItemId String
|
||||||
|
snapshotId String?
|
||||||
|
changeType PropertyChangeType
|
||||||
|
severity PropertyChangeSeverity @default(info)
|
||||||
|
details Json?
|
||||||
|
detectedAt DateTime @default(now())
|
||||||
|
|
||||||
|
propertyWatchlistItem PropertyWatchlistItem @relation(fields: [propertyWatchlistItemId], references: [id], onDelete: Cascade)
|
||||||
|
snapshot PropertySnapshot? @relation("SnapshotChanges", fields: [snapshotId], references: [id])
|
||||||
|
|
||||||
|
@@index([propertyWatchlistItemId])
|
||||||
|
@@index([snapshotId])
|
||||||
|
@@index([changeType])
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// Info Broker Removal Models
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
enum BrokerCategory {
|
||||||
|
people_search
|
||||||
|
background_check
|
||||||
|
public_records
|
||||||
|
reverse_lookup
|
||||||
|
social_media
|
||||||
|
}
|
||||||
|
|
||||||
|
enum RemovalMethod {
|
||||||
|
automated
|
||||||
|
manual_form
|
||||||
|
email
|
||||||
|
phone
|
||||||
|
mail
|
||||||
|
none
|
||||||
|
}
|
||||||
|
|
||||||
|
enum RemovalStatus {
|
||||||
|
pending
|
||||||
|
submitted
|
||||||
|
in_progress
|
||||||
|
completed
|
||||||
|
failed
|
||||||
|
rejected
|
||||||
|
}
|
||||||
|
|
||||||
|
model InfoBroker {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
name String
|
||||||
|
domain String @unique
|
||||||
|
category BrokerCategory
|
||||||
|
removalMethod RemovalMethod
|
||||||
|
removalUrl String?
|
||||||
|
requiresAccount Boolean @default(false)
|
||||||
|
requiresVerification Boolean @default(false)
|
||||||
|
estimatedDays Int @default(14)
|
||||||
|
isActive Boolean @default(true)
|
||||||
|
|
||||||
|
removalRequests RemovalRequest[]
|
||||||
|
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @default(now()) @updatedAt
|
||||||
|
|
||||||
|
@@index([category])
|
||||||
|
@@index([isActive])
|
||||||
|
@@index([removalMethod])
|
||||||
|
}
|
||||||
|
|
||||||
|
model RemovalRequest {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
subscriptionId String
|
||||||
|
brokerId String
|
||||||
|
status RemovalStatus @default(pending)
|
||||||
|
personalInfo Json // { fullName, email?, phone?, address?, dob? }
|
||||||
|
method RemovalMethod
|
||||||
|
attempts Int @default(0)
|
||||||
|
nextRetryAt DateTime?
|
||||||
|
submittedAt DateTime?
|
||||||
|
completedAt DateTime?
|
||||||
|
error String?
|
||||||
|
notes String?
|
||||||
|
metadata Json? // Broker response data, tracking info
|
||||||
|
|
||||||
|
broker InfoBroker @relation(fields: [brokerId], references: [id])
|
||||||
|
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @default(now()) @updatedAt
|
||||||
|
|
||||||
|
@@index([subscriptionId])
|
||||||
|
@@index([brokerId])
|
||||||
|
@@index([status])
|
||||||
|
@@index([submittedAt])
|
||||||
|
@@index([subscriptionId, status])
|
||||||
|
}
|
||||||
|
|
||||||
|
model BrokerListing {
|
||||||
|
id String @id @default(uuid())
|
||||||
|
subscriptionId String
|
||||||
|
brokerId String
|
||||||
|
removalRequestId String?
|
||||||
|
url String
|
||||||
|
dataFound Json // Fields found on the listing
|
||||||
|
screenshotUrl String?
|
||||||
|
isRemoved Boolean @default(false)
|
||||||
|
removedAt DateTime?
|
||||||
|
|
||||||
|
removalRequest RemovalRequest? @relation(fields: [removalRequestId], references: [id])
|
||||||
|
|
||||||
|
scannedAt DateTime @default(now())
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @default(now()) @updatedAt
|
||||||
|
|
||||||
|
@@index([subscriptionId])
|
||||||
|
@@index([brokerId])
|
||||||
|
@@index([removalRequestId])
|
||||||
|
@@index([isRemoved])
|
||||||
|
@@index([subscriptionId, isRemoved])
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ export const AlertSource = {
|
|||||||
SPAMSHIELD: "SPAMSHIELD",
|
SPAMSHIELD: "SPAMSHIELD",
|
||||||
VOICEPRINT: "VOICEPRINT",
|
VOICEPRINT: "VOICEPRINT",
|
||||||
CALL_ANALYSIS: "CALL_ANALYSIS",
|
CALL_ANALYSIS: "CALL_ANALYSIS",
|
||||||
|
INFO_BROKER: "INFO_BROKER",
|
||||||
} as const;
|
} as const;
|
||||||
export type AlertSource = (typeof AlertSource)[keyof typeof AlertSource];
|
export type AlertSource = (typeof AlertSource)[keyof typeof AlertSource];
|
||||||
|
|
||||||
@@ -39,6 +40,9 @@ export const AlertCategory = {
|
|||||||
CALL_QUALITY: "CALL_QUALITY",
|
CALL_QUALITY: "CALL_QUALITY",
|
||||||
CALL_ANOMALY: "CALL_ANOMALY",
|
CALL_ANOMALY: "CALL_ANOMALY",
|
||||||
CALL_EVENT: "CALL_EVENT",
|
CALL_EVENT: "CALL_EVENT",
|
||||||
|
HOME_TITLE: "HOME_TITLE",
|
||||||
|
INFO_BROKER_LISTING: "INFO_BROKER_LISTING",
|
||||||
|
INFO_BROKER_REMOVAL: "INFO_BROKER_REMOVAL",
|
||||||
} as const;
|
} as const;
|
||||||
export type AlertCategory = (typeof AlertCategory)[keyof typeof AlertCategory];
|
export type AlertCategory = (typeof AlertCategory)[keyof typeof AlertCategory];
|
||||||
|
|
||||||
@@ -280,6 +284,7 @@ export { generateRequestId, extractOrGenerateRequestId } from "./requestId";
|
|||||||
export const ReportType = {
|
export const ReportType = {
|
||||||
MONTHLY_PLUS: "MONTHLY_PLUS",
|
MONTHLY_PLUS: "MONTHLY_PLUS",
|
||||||
ANNUAL_PREMIUM: "ANNUAL_PREMIUM",
|
ANNUAL_PREMIUM: "ANNUAL_PREMIUM",
|
||||||
|
WEEKLY_DIGEST: "WEEKLY_DIGEST",
|
||||||
} as const;
|
} as const;
|
||||||
export type ReportType = (typeof ReportType)[keyof typeof ReportType];
|
export type ReportType = (typeof ReportType)[keyof typeof ReportType];
|
||||||
|
|
||||||
@@ -365,3 +370,93 @@ export interface SecurityReportOutput {
|
|||||||
createdAt: Date;
|
createdAt: Date;
|
||||||
deliveredAt?: Date;
|
deliveredAt?: Date;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// Info Broker Removal Types
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
export const BrokerStatus = {
|
||||||
|
ACTIVE: "ACTIVE",
|
||||||
|
INACTIVE: "INACTIVE",
|
||||||
|
UNKNOWN: "UNKNOWN",
|
||||||
|
} as const;
|
||||||
|
export type BrokerStatus = (typeof BrokerStatus)[keyof typeof BrokerStatus];
|
||||||
|
|
||||||
|
export const RemovalStatus = {
|
||||||
|
PENDING: "PENDING",
|
||||||
|
SUBMITTED: "SUBMITTED",
|
||||||
|
IN_PROGRESS: "IN_PROGRESS",
|
||||||
|
COMPLETED: "COMPLETED",
|
||||||
|
FAILED: "FAILED",
|
||||||
|
REJECTED: "REJECTED",
|
||||||
|
} as const;
|
||||||
|
export type RemovalStatus = (typeof RemovalStatus)[keyof typeof RemovalStatus];
|
||||||
|
|
||||||
|
export const RemovalMethod = {
|
||||||
|
AUTOMATED: "AUTOMATED",
|
||||||
|
MANUAL_FORM: "MANUAL_FORM",
|
||||||
|
EMAIL: "EMAIL",
|
||||||
|
PHONE: "PHONE",
|
||||||
|
MAIL: "MAIL",
|
||||||
|
NONE: "NONE",
|
||||||
|
} as const;
|
||||||
|
export type RemovalMethod = (typeof RemovalMethod)[keyof typeof RemovalMethod];
|
||||||
|
|
||||||
|
export interface BrokerDefinition {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
domain: string;
|
||||||
|
category: string;
|
||||||
|
removalMethod: RemovalMethod;
|
||||||
|
removalUrl?: string;
|
||||||
|
requiresAccount: boolean;
|
||||||
|
requiresVerification: boolean;
|
||||||
|
estimatedDays: number;
|
||||||
|
isActive: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RemovalRequestInput {
|
||||||
|
brokerId: string;
|
||||||
|
personalInfo: {
|
||||||
|
fullName: string;
|
||||||
|
email?: string;
|
||||||
|
phone?: string;
|
||||||
|
address?: {
|
||||||
|
street?: string;
|
||||||
|
city?: string;
|
||||||
|
state?: string;
|
||||||
|
zip?: string;
|
||||||
|
};
|
||||||
|
dob?: string;
|
||||||
|
};
|
||||||
|
notes?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RemovalRequestOutput {
|
||||||
|
id: string;
|
||||||
|
brokerId: string;
|
||||||
|
brokerName: string;
|
||||||
|
status: RemovalStatus;
|
||||||
|
submittedAt?: Date;
|
||||||
|
completedAt?: Date;
|
||||||
|
method: RemovalMethod;
|
||||||
|
attempts: number;
|
||||||
|
nextRetryAt?: Date;
|
||||||
|
notes?: string;
|
||||||
|
error?: string;
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BrokerScanResult {
|
||||||
|
brokerId: string;
|
||||||
|
brokerName: string;
|
||||||
|
listingsFound: number;
|
||||||
|
listings: Array<{
|
||||||
|
id: string;
|
||||||
|
url: string;
|
||||||
|
dataFound: Record<string, string>;
|
||||||
|
screenshotUrl?: string;
|
||||||
|
}>;
|
||||||
|
scannedAt: Date;
|
||||||
|
}
|
||||||
|
|||||||
25
services/removebrokers/package.json
Normal file
25
services/removebrokers/package.json
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"name": "@shieldai/removebrokers",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"main": "./dist/index.js",
|
||||||
|
"types": "./dist/index.js",
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc",
|
||||||
|
"test": "vitest run",
|
||||||
|
"test:coverage": "vitest run --coverage",
|
||||||
|
"lint": "eslint src/"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@shieldai/db": "workspace:*",
|
||||||
|
"@shieldai/types": "workspace:*",
|
||||||
|
"@shieldai/shared-notifications": "workspace:*",
|
||||||
|
"node-cache": "^5.1.2"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"vitest": "^4.1.5",
|
||||||
|
"@vitest/coverage-v8": "^4.1.5"
|
||||||
|
},
|
||||||
|
"exports": {
|
||||||
|
".": "./src/index.ts"
|
||||||
|
}
|
||||||
|
}
|
||||||
69
services/removebrokers/src/BrokerAlertPipeline.ts
Normal file
69
services/removebrokers/src/BrokerAlertPipeline.ts
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
import { AlertSource, AlertCategory, Severity, EntityType } from "@shieldai/types";
|
||||||
|
|
||||||
|
export interface BrokerAlertInput {
|
||||||
|
userId: string;
|
||||||
|
brokerName: string;
|
||||||
|
brokerId: string;
|
||||||
|
category: AlertCategory;
|
||||||
|
severity: Severity;
|
||||||
|
title: string;
|
||||||
|
description: string;
|
||||||
|
entities: Array<{ type: EntityType; value: string }>;
|
||||||
|
metadata?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class BrokerAlertPipeline {
|
||||||
|
async sendListingFoundAlert(input: BrokerAlertInput) {
|
||||||
|
const alert = {
|
||||||
|
source: AlertSource.INFO_BROKER,
|
||||||
|
category: AlertCategory.INFO_BROKER_LISTING,
|
||||||
|
severity: input.severity,
|
||||||
|
userId: input.userId,
|
||||||
|
title: input.title,
|
||||||
|
description: input.description,
|
||||||
|
entities: input.entities,
|
||||||
|
sourceAlertId: `broker_listing_${input.brokerId}_${Date.now()}`,
|
||||||
|
payload: {
|
||||||
|
brokerId: input.brokerId,
|
||||||
|
brokerName: input.brokerName,
|
||||||
|
...input.metadata,
|
||||||
|
},
|
||||||
|
timestamp: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
return this.normalizeAndSend(alert);
|
||||||
|
}
|
||||||
|
|
||||||
|
async sendRemovalStatusAlert(input: BrokerAlertInput) {
|
||||||
|
const alert = {
|
||||||
|
source: AlertSource.INFO_BROKER,
|
||||||
|
category: AlertCategory.INFO_BROKER_REMOVAL,
|
||||||
|
severity: input.severity,
|
||||||
|
userId: input.userId,
|
||||||
|
title: input.title,
|
||||||
|
description: input.description,
|
||||||
|
entities: input.entities,
|
||||||
|
sourceAlertId: `broker_removal_${input.brokerId}_${Date.now()}`,
|
||||||
|
payload: {
|
||||||
|
brokerId: input.brokerId,
|
||||||
|
brokerName: input.brokerName,
|
||||||
|
...input.metadata,
|
||||||
|
},
|
||||||
|
timestamp: new Date(),
|
||||||
|
};
|
||||||
|
|
||||||
|
return this.normalizeAndSend(alert);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async normalizeAndSend(alert: any) {
|
||||||
|
try {
|
||||||
|
const { correlationPipeline } = await import("@shieldai/correlation");
|
||||||
|
return correlationPipeline.normalizeAlert(alert);
|
||||||
|
} catch {
|
||||||
|
console.error("[BrokerAlert] Failed to send alert:", alert.sourceAlertId);
|
||||||
|
return alert;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const brokerAlertPipeline = new BrokerAlertPipeline();
|
||||||
145
services/removebrokers/src/RemoveBrokersScheduler.ts
Normal file
145
services/removebrokers/src/RemoveBrokersScheduler.ts
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
import prisma from "@shieldai/db";
|
||||||
|
import { RemovalStatus } from "@shieldai/types";
|
||||||
|
import { removeBrokersService } from "./RemoveBrokersService";
|
||||||
|
|
||||||
|
const DEFAULT_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||||
|
const VERIFICATION_DELAY_MS = 7 * 24 * 60 * 60 * 1000; // 7 days after submission
|
||||||
|
|
||||||
|
export class RemoveBrokersScheduler {
|
||||||
|
private intervalId: NodeJS.Timeout | null = null;
|
||||||
|
private intervalMs: number;
|
||||||
|
|
||||||
|
constructor(intervalMs: number = DEFAULT_SCAN_INTERVAL_MS) {
|
||||||
|
this.intervalMs = intervalMs;
|
||||||
|
}
|
||||||
|
|
||||||
|
isRunning(): boolean {
|
||||||
|
return this.intervalId !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
start() {
|
||||||
|
if (this.isRunning()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.intervalId = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
await this.runCycle();
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[RemoveBrokers] Scheduler cycle error:", error);
|
||||||
|
}
|
||||||
|
}, this.intervalMs);
|
||||||
|
|
||||||
|
console.log("[RemoveBrokers] Scheduler started");
|
||||||
|
}
|
||||||
|
|
||||||
|
stop() {
|
||||||
|
if (this.intervalId) {
|
||||||
|
clearInterval(this.intervalId);
|
||||||
|
this.intervalId = null;
|
||||||
|
console.log("[RemoveBrokers] Scheduler stopped");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async runCycle() {
|
||||||
|
console.log("[RemoveBrokers] Running scheduler cycle...");
|
||||||
|
|
||||||
|
const results = {
|
||||||
|
processed: 0,
|
||||||
|
verified: 0,
|
||||||
|
errors: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const processResults = await removeBrokersService.processPendingRequests();
|
||||||
|
results.processed = processResults.length;
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[RemoveBrokers] Process pending error:", error);
|
||||||
|
results.errors++;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const verifyResults = await this.verifyCompletedRemovals();
|
||||||
|
results.verified = verifyResults.length;
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[RemoveBrokers] Verify completions error:", error);
|
||||||
|
results.errors++;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[RemoveBrokers] Cycle complete: ${JSON.stringify(results)}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
async runScan(subscriptionId: string) {
|
||||||
|
const subscription = await prisma.subscription.findUnique({
|
||||||
|
where: { id: subscriptionId },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!subscription) {
|
||||||
|
throw new Error("Subscription not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
const user = await prisma.user.findUnique({
|
||||||
|
where: { id: subscription.userId },
|
||||||
|
select: { name: true, email: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!user?.name) {
|
||||||
|
throw new Error("User name required for scanning");
|
||||||
|
}
|
||||||
|
|
||||||
|
const personalInfo = {
|
||||||
|
fullName: user.name,
|
||||||
|
email: user.email,
|
||||||
|
};
|
||||||
|
|
||||||
|
const results = await removeBrokersService.scanForListings(
|
||||||
|
subscriptionId,
|
||||||
|
personalInfo,
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
subscriptionId,
|
||||||
|
brokersScanned: results.length,
|
||||||
|
listingsFound: results.filter((r) => r.found).length,
|
||||||
|
results,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async verifyCompletedRemovals() {
|
||||||
|
const submitted = await prisma.removalRequest.findMany({
|
||||||
|
where: {
|
||||||
|
status: RemovalStatus.SUBMITTED,
|
||||||
|
submittedAt: {
|
||||||
|
lte: new Date(Date.now() - VERIFICATION_DELAY_MS),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = [];
|
||||||
|
|
||||||
|
for (const request of submitted) {
|
||||||
|
try {
|
||||||
|
const verification = await removeBrokersService.verifyRemoval(
|
||||||
|
request.id,
|
||||||
|
);
|
||||||
|
results.push({
|
||||||
|
requestId: request.id,
|
||||||
|
...verification,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error(
|
||||||
|
`[RemoveBrokers] Verification error for ${request.id}:`,
|
||||||
|
error,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const removeBrokersScheduler = new RemoveBrokersScheduler();
|
||||||
478
services/removebrokers/src/RemoveBrokersService.ts
Normal file
478
services/removebrokers/src/RemoveBrokersService.ts
Normal file
@@ -0,0 +1,478 @@
|
|||||||
|
import prisma from "@shieldai/db";
|
||||||
|
import { RemovalStatus, RemovalMethod } from "@shieldai/types";
|
||||||
|
import { getBrokerById, getActiveBrokers } from "./brokerRegistry";
|
||||||
|
import type { PersonalInfo, RemovalJob, BrokerEntry } from "./types";
|
||||||
|
import { MAX_REMOVAL_ATTEMPTS, RETRY_DELAY_MS } from "./types";
|
||||||
|
|
||||||
|
export class RemoveBrokersService {
|
||||||
|
async scanForListings(subscriptionId: string, personalInfo: PersonalInfo) {
|
||||||
|
const brokers = getActiveBrokers();
|
||||||
|
const results = [];
|
||||||
|
|
||||||
|
for (const broker of brokers) {
|
||||||
|
const existingListing = await prisma.brokerListing.findFirst({
|
||||||
|
where: {
|
||||||
|
subscriptionId,
|
||||||
|
brokerId: broker.id,
|
||||||
|
isRemoved: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (existingListing) {
|
||||||
|
results.push({
|
||||||
|
brokerId: broker.id,
|
||||||
|
brokerName: broker.name,
|
||||||
|
found: true,
|
||||||
|
listingId: existingListing.id,
|
||||||
|
url: existingListing.url,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const found = await this.checkBrokerListing(broker, personalInfo);
|
||||||
|
if (found) {
|
||||||
|
const listing = await prisma.brokerListing.create({
|
||||||
|
data: {
|
||||||
|
subscriptionId,
|
||||||
|
brokerId: broker.id,
|
||||||
|
url: found.url,
|
||||||
|
dataFound: found.dataFound,
|
||||||
|
isRemoved: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
results.push({
|
||||||
|
brokerId: broker.id,
|
||||||
|
brokerName: broker.name,
|
||||||
|
found: true,
|
||||||
|
listingId: listing.id,
|
||||||
|
url: found.url,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
results.push({
|
||||||
|
brokerId: broker.id,
|
||||||
|
brokerName: broker.name,
|
||||||
|
found: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
async createRemovalRequest(
|
||||||
|
subscriptionId: string,
|
||||||
|
brokerId: string,
|
||||||
|
personalInfo: PersonalInfo,
|
||||||
|
notes?: string,
|
||||||
|
) {
|
||||||
|
const broker = getBrokerById(brokerId);
|
||||||
|
if (!broker) {
|
||||||
|
throw new Error(`Broker not found: ${brokerId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = await prisma.removalRequest.findFirst({
|
||||||
|
where: {
|
||||||
|
subscriptionId,
|
||||||
|
brokerId,
|
||||||
|
status: { in: [RemovalStatus.PENDING, RemovalStatus.SUBMITTED, RemovalStatus.IN_PROGRESS] },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (existing) {
|
||||||
|
throw new Error(`Active removal request already exists for ${broker.name}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const request = await prisma.removalRequest.create({
|
||||||
|
data: {
|
||||||
|
subscriptionId,
|
||||||
|
brokerId,
|
||||||
|
status: RemovalStatus.PENDING,
|
||||||
|
personalInfo: personalInfo as any,
|
||||||
|
method: broker.removalMethod,
|
||||||
|
notes,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
async submitRemoval(job: RemovalJob): Promise<boolean> {
|
||||||
|
const broker = getBrokerById(job.brokerId);
|
||||||
|
if (!broker) {
|
||||||
|
throw new Error(`Broker not found: ${job.brokerId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (job.method) {
|
||||||
|
case RemovalMethod.AUTOMATED:
|
||||||
|
return await this.submitAutomatedRemoval(job, broker);
|
||||||
|
case RemovalMethod.MANUAL_FORM:
|
||||||
|
return await this.submitManualFormRemoval(job, broker);
|
||||||
|
case RemovalMethod.EMAIL:
|
||||||
|
return await this.submitEmailRemoval(job, broker);
|
||||||
|
default:
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async processPendingRequests() {
|
||||||
|
const pending = await prisma.removalRequest.findMany({
|
||||||
|
where: {
|
||||||
|
status: RemovalStatus.PENDING,
|
||||||
|
OR: [
|
||||||
|
{ nextRetryAt: null },
|
||||||
|
{ nextRetryAt: { lte: new Date() } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = [];
|
||||||
|
|
||||||
|
for (const request of pending) {
|
||||||
|
try {
|
||||||
|
await prisma.removalRequest.update({
|
||||||
|
where: { id: request.id },
|
||||||
|
data: { status: RemovalStatus.IN_PROGRESS },
|
||||||
|
});
|
||||||
|
|
||||||
|
const job: RemovalJob = {
|
||||||
|
requestId: request.id,
|
||||||
|
brokerId: request.brokerId,
|
||||||
|
brokerName: request.brokerId,
|
||||||
|
personalInfo: request.personalInfo as PersonalInfo,
|
||||||
|
method: request.method,
|
||||||
|
attempt: request.attempts + 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
const success = await this.submitRemoval(job);
|
||||||
|
|
||||||
|
if (success) {
|
||||||
|
await prisma.removalRequest.update({
|
||||||
|
where: { id: request.id },
|
||||||
|
data: {
|
||||||
|
status: RemovalStatus.SUBMITTED,
|
||||||
|
attempts: request.attempts + 1,
|
||||||
|
submittedAt: new Date(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
results.push({ requestId: request.id, status: "submitted" });
|
||||||
|
} else if (request.attempts + 1 >= MAX_REMOVAL_ATTEMPTS) {
|
||||||
|
await prisma.removalRequest.update({
|
||||||
|
where: { id: request.id },
|
||||||
|
data: {
|
||||||
|
status: RemovalStatus.FAILED,
|
||||||
|
attempts: request.attempts + 1,
|
||||||
|
error: "Max attempts reached",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
results.push({ requestId: request.id, status: "failed" });
|
||||||
|
} else {
|
||||||
|
await prisma.removalRequest.update({
|
||||||
|
where: { id: request.id },
|
||||||
|
data: {
|
||||||
|
attempts: request.attempts + 1,
|
||||||
|
nextRetryAt: new Date(Date.now() + RETRY_DELAY_MS),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
results.push({ requestId: request.id, status: "retry_scheduled" });
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : "Unknown error";
|
||||||
|
await prisma.removalRequest.update({
|
||||||
|
where: { id: request.id },
|
||||||
|
data: {
|
||||||
|
status: RemovalStatus.PENDING,
|
||||||
|
error: message,
|
||||||
|
nextRetryAt: new Date(Date.now() + RETRY_DELAY_MS),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
results.push({ requestId: request.id, status: "error", error: message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
async verifyRemoval(requestId: string) {
|
||||||
|
const request = await prisma.removalRequest.findUnique({
|
||||||
|
where: { id: requestId },
|
||||||
|
include: { broker: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!request) {
|
||||||
|
throw new Error(`Removal request not found: ${requestId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const personalInfo = request.personalInfo as PersonalInfo;
|
||||||
|
const stillListed = await this.checkBrokerListing(
|
||||||
|
request.broker,
|
||||||
|
personalInfo,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!stillListed) {
|
||||||
|
await prisma.removalRequest.update({
|
||||||
|
where: { id: requestId },
|
||||||
|
data: {
|
||||||
|
status: RemovalStatus.COMPLETED,
|
||||||
|
completedAt: new Date(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await prisma.brokerListing.updateMany({
|
||||||
|
where: {
|
||||||
|
removalRequestId: requestId,
|
||||||
|
isRemoved: false,
|
||||||
|
},
|
||||||
|
data: {
|
||||||
|
isRemoved: true,
|
||||||
|
removedAt: new Date(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return { completed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { completed: false, stillListed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
async getRemovalStatus(subscriptionId: string) {
|
||||||
|
const requests = await prisma.removalRequest.findMany({
|
||||||
|
where: { subscriptionId },
|
||||||
|
include: { broker: true },
|
||||||
|
orderBy: { updatedAt: "desc" },
|
||||||
|
});
|
||||||
|
|
||||||
|
return requests.map((r: any) => ({
|
||||||
|
id: r.id,
|
||||||
|
brokerId: r.brokerId,
|
||||||
|
brokerName: r.broker.name,
|
||||||
|
status: r.status,
|
||||||
|
method: r.method,
|
||||||
|
attempts: r.attempts,
|
||||||
|
submittedAt: r.submittedAt,
|
||||||
|
completedAt: r.completedAt,
|
||||||
|
error: r.error,
|
||||||
|
createdAt: r.createdAt,
|
||||||
|
updatedAt: r.updatedAt,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
async getAvailableBrokers(): Promise<BrokerEntry[]> {
|
||||||
|
return getActiveBrokers();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Private methods ----
|
||||||
|
|
||||||
|
private async checkBrokerListing(
|
||||||
|
broker: BrokerEntry,
|
||||||
|
personalInfo: PersonalInfo,
|
||||||
|
) {
|
||||||
|
const searchUrl = this.buildSearchUrl(broker, personalInfo);
|
||||||
|
try {
|
||||||
|
const response = await fetch(searchUrl, {
|
||||||
|
headers: {
|
||||||
|
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(10000),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const html = await response.text();
|
||||||
|
const listingUrl = this.extractListingUrl(html, searchUrl);
|
||||||
|
|
||||||
|
if (!listingUrl) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const dataFound = this.extractPersonalData(html, personalInfo);
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: listingUrl,
|
||||||
|
dataFound,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildSearchUrl(broker: BrokerEntry, info: PersonalInfo): string {
|
||||||
|
const nameParts = info.fullName.split(" ");
|
||||||
|
const firstName = nameParts[0] || "";
|
||||||
|
const lastName = nameParts.slice(1).join(" ") || "";
|
||||||
|
|
||||||
|
const urlMap: Record<string, (f: string, l: string, s: string) => string> = {
|
||||||
|
whitepages: (f, l) =>
|
||||||
|
`https://www.whitepages.com/people/${f.toLowerCase()}-${l.toLowerCase()}`,
|
||||||
|
spokeo: (f, l) =>
|
||||||
|
`https://www.spokeo.com/search?q=${encodeURIComponent(info.fullName)}`,
|
||||||
|
truepeoplesearch: (f, l) =>
|
||||||
|
`https://www.truepeoplesearch.com/name/${encodeURIComponent(info.fullName)}`,
|
||||||
|
peoplefinders: (f, l) =>
|
||||||
|
`https://www.peoplefinders.com/results?name=${encodeURIComponent(info.fullName)}`,
|
||||||
|
thatsmth: (f, l) =>
|
||||||
|
`https://thatsmth.com/name/${encodeURIComponent(info.fullName)}`,
|
||||||
|
fastpeoplesearch: (f, l) =>
|
||||||
|
`https://www.fastpeoplesearch.com/name/${encodeURIComponent(info.fullName)}`,
|
||||||
|
};
|
||||||
|
|
||||||
|
const builder = urlMap[broker.id];
|
||||||
|
if (builder) {
|
||||||
|
return builder(firstName, lastName, info.address?.state || "");
|
||||||
|
}
|
||||||
|
|
||||||
|
return `https://${broker.domain}/search?q=${encodeURIComponent(info.fullName)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractListingUrl(html: string, searchUrl: string): string | null {
|
||||||
|
const profilePatterns = [
|
||||||
|
/href="([^"]*\/people\/[^"]+)"/,
|
||||||
|
/href="([^"]*\/profile\/[^"]+)"/,
|
||||||
|
/href="([^"]*\/results\/[^"]+)"/,
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const pattern of profilePatterns) {
|
||||||
|
const match = html.match(pattern);
|
||||||
|
if (match) {
|
||||||
|
let url = match[1];
|
||||||
|
if (url.startsWith("/")) {
|
||||||
|
const urlObj = new URL(searchUrl);
|
||||||
|
url = `${urlObj.protocol}//${urlObj.host}${url}`;
|
||||||
|
}
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractPersonalData(html: string, _info: PersonalInfo): Record<string, string> {
|
||||||
|
const data: Record<string, string> = {};
|
||||||
|
|
||||||
|
const phonePattern = /\b(\d{3}[-.]?\d{3}[-.]?\d{4})\b/;
|
||||||
|
const phoneMatch = html.match(phonePattern);
|
||||||
|
if (phoneMatch) {
|
||||||
|
data.phoneNumber = phoneMatch[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
const addressPattern = /(\d+\s+[A-Za-z\s]+,\s*[A-Z]{2}\s*\d{5})/;
|
||||||
|
const addressMatch = html.match(addressPattern);
|
||||||
|
if (addressMatch) {
|
||||||
|
data.address = addressMatch[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
const emailPattern = /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/;
|
||||||
|
const emailMatch = html.match(emailPattern);
|
||||||
|
if (emailMatch) {
|
||||||
|
data.email = emailMatch[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async submitAutomatedRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||||
|
if (!broker.removalUrl) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(broker.removalUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/x-www-form-urlencoded",
|
||||||
|
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||||
|
},
|
||||||
|
body: new URLSearchParams({
|
||||||
|
full_name: job.personalInfo.fullName,
|
||||||
|
email: job.personalInfo.email || "",
|
||||||
|
phone: job.personalInfo.phone || "",
|
||||||
|
address: job.personalInfo.address
|
||||||
|
? `${job.personalInfo.address.street || ""}, ${job.personalInfo.address.city || ""}, ${job.personalInfo.address.state || ""} ${job.personalInfo.address.zip || ""}`.trim()
|
||||||
|
: "",
|
||||||
|
dob: job.personalInfo.dob || "",
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(30000),
|
||||||
|
});
|
||||||
|
|
||||||
|
return response.ok || response.status === 302;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async submitManualFormRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||||
|
if (!broker.removalUrl) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(broker.removalUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
name: job.personalInfo.fullName,
|
||||||
|
email: job.personalInfo.email,
|
||||||
|
phone: job.personalInfo.phone,
|
||||||
|
address: job.personalInfo.address,
|
||||||
|
reason: "privacy_removal",
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(30000),
|
||||||
|
});
|
||||||
|
|
||||||
|
return response.ok || response.status === 302;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async submitEmailRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||||
|
const emailBody = this.generateEmailRemovalRequest(job, broker);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const emailServiceUrl = process.env.REMOVAL_EMAIL_SERVICE_URL;
|
||||||
|
if (!emailServiceUrl) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await fetch(emailServiceUrl, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
to: `privacy@${broker.domain}`,
|
||||||
|
subject: `Data Removal Request - ${job.personalInfo.fullName}`,
|
||||||
|
body: emailBody,
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(15000),
|
||||||
|
});
|
||||||
|
|
||||||
|
return response.ok;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private generateEmailRemovalRequest(job: RemovalJob, broker: BrokerEntry): string {
|
||||||
|
return `Dear ${broker.name} Privacy Team,
|
||||||
|
|
||||||
|
I am requesting the removal of my personal information from your website (${broker.domain}).
|
||||||
|
|
||||||
|
My Details:
|
||||||
|
- Full Name: ${job.personalInfo.fullName}
|
||||||
|
${job.personalInfo.email ? `- Email: ${job.personalInfo.email}` : ""}
|
||||||
|
${job.personalInfo.phone ? `- Phone: ${job.personalInfo.phone}` : ""}
|
||||||
|
${job.personalInfo.address ? `- Address: ${job.personalInfo.address.street || ""}, ${job.personalInfo.address.city || ""}, ${job.personalInfo.address.state || ""} ${job.personalInfo.address.zip || ""}`.trim() : ""}
|
||||||
|
${job.personalInfo.dob ? `- Date of Birth: ${job.personalInfo.dob}` : ""}
|
||||||
|
|
||||||
|
I do not consent to the publication of my personal information on your site. Please process this removal request within 30 days as required by applicable privacy laws.
|
||||||
|
|
||||||
|
Thank you,
|
||||||
|
${job.personalInfo.fullName}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const removeBrokersService = new RemoveBrokersService();
|
||||||
261
services/removebrokers/src/brokerRegistry.ts
Normal file
261
services/removebrokers/src/brokerRegistry.ts
Normal file
@@ -0,0 +1,261 @@
|
|||||||
|
import { RemovalMethod } from "@shieldai/types";
|
||||||
|
import type { BrokerEntry } from "./types";
|
||||||
|
|
||||||
|
export const BROKER_REGISTRY: BrokerEntry[] = [
|
||||||
|
{
|
||||||
|
id: "whitepages",
|
||||||
|
name: "Whitepages",
|
||||||
|
domain: "whitepages.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://www.whitepages.com/optout",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 7,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "spokeo",
|
||||||
|
name: "Spokeo",
|
||||||
|
domain: "spokeo.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://www.spokeo.com/privacy/removal-request",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "truepeoplesearch",
|
||||||
|
name: "TruePeopleSearch",
|
||||||
|
domain: "truepeoplesearch.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.AUTOMATED,
|
||||||
|
removalUrl: "https://www.truepeoplesearch.com/remove-your-info",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 3,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "peoplefinders",
|
||||||
|
name: "PeopleFinders",
|
||||||
|
domain: "peoplefinders.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://www.peoplefinders.com/privacy-policy",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "thatsmth",
|
||||||
|
name: "That's Them",
|
||||||
|
domain: "thatsmth.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.AUTOMATED,
|
||||||
|
removalUrl: "https://thatsmth.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 7,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "fastpeoplesearch",
|
||||||
|
name: "FastPeopleSearch",
|
||||||
|
domain: "fastpeoplesearch.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.AUTOMATED,
|
||||||
|
removalUrl: "https://www.fastpeoplesearch.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 5,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "backgroundcheck",
|
||||||
|
name: "BackgroundCheck",
|
||||||
|
domain: "backgroundcheck.com",
|
||||||
|
category: "background_check",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://www.backgroundcheck.com/removal",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "freepeopledirectory",
|
||||||
|
name: "Free People Directory",
|
||||||
|
domain: "freepeopledirectory.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.AUTOMATED,
|
||||||
|
removalUrl: "https://freepeopledirectory.com/optout",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 7,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "radaris",
|
||||||
|
name: "Radaris",
|
||||||
|
domain: "radaris.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.EMAIL,
|
||||||
|
removalUrl: undefined,
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 30,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "zynda",
|
||||||
|
name: "Zynda",
|
||||||
|
domain: "zynda.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://zynda.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "addressinator",
|
||||||
|
name: "Addressinator",
|
||||||
|
domain: "addressinator.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://addressinator.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "familytree Now",
|
||||||
|
name: "FamilyTree Now",
|
||||||
|
domain: "familytreenow.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.EMAIL,
|
||||||
|
removalUrl: undefined,
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 30,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "accuratebackground",
|
||||||
|
name: "Accurate Background",
|
||||||
|
domain: "accuratebackground.com",
|
||||||
|
category: "background_check",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://www.accuratebackground.com/optout",
|
||||||
|
requiresAccount: true,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 30,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "instantcheckmate",
|
||||||
|
name: "Instant Checkmate",
|
||||||
|
domain: "instantcheckmate.com",
|
||||||
|
category: "background_check",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://www.instantcheckmate.com/opt-out",
|
||||||
|
requiresAccount: true,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 30,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "pthree",
|
||||||
|
name: "P3 (People Finders)",
|
||||||
|
domain: "pthree.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://www.pthree.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "sortedbee",
|
||||||
|
name: "Sorted Bee",
|
||||||
|
domain: "sortedbee.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://www.sortedbee.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "ussearch",
|
||||||
|
name: "US Search",
|
||||||
|
domain: "ussearch.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.AUTOMATED,
|
||||||
|
removalUrl: "https://www.ussearch.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 7,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "tellme",
|
||||||
|
name: "Tell me Online Info",
|
||||||
|
domain: "tellmeonlineinfo.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||||
|
removalUrl: "https://tellmeonlineinfo.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "synpeople",
|
||||||
|
name: "Synpeople",
|
||||||
|
domain: "synpeople.com",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.AUTOMATED,
|
||||||
|
removalUrl: "https://www.synpeople.com/opt-out",
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: true,
|
||||||
|
estimatedDays: 7,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "atomdata",
|
||||||
|
name: "Atom Data",
|
||||||
|
domain: "atomdata.xyz",
|
||||||
|
category: "people_search",
|
||||||
|
removalMethod: RemovalMethod.EMAIL,
|
||||||
|
removalUrl: undefined,
|
||||||
|
requiresAccount: false,
|
||||||
|
requiresVerification: false,
|
||||||
|
estimatedDays: 14,
|
||||||
|
isActive: true,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
export function getBrokerById(id: string): BrokerEntry | undefined {
|
||||||
|
return BROKER_REGISTRY.find((b) => b.id === id);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getActiveBrokers(): BrokerEntry[] {
|
||||||
|
return BROKER_REGISTRY.filter((b) => b.isActive);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getBrokersByCategory(category: string): BrokerEntry[] {
|
||||||
|
return BROKER_REGISTRY.filter((b) => b.category === category);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getBrokersByMethod(method: RemovalMethod): BrokerEntry[] {
|
||||||
|
return BROKER_REGISTRY.filter((b) => b.removalMethod === method);
|
||||||
|
}
|
||||||
8
services/removebrokers/src/index.ts
Normal file
8
services/removebrokers/src/index.ts
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
export { removeBrokersService } from "./RemoveBrokersService";
|
||||||
|
export { RemoveBrokersService } from "./RemoveBrokersService";
|
||||||
|
export { removeBrokersScheduler } from "./RemoveBrokersScheduler";
|
||||||
|
export { RemoveBrokersScheduler } from "./RemoveBrokersScheduler";
|
||||||
|
export { brokerAlertPipeline } from "./BrokerAlertPipeline";
|
||||||
|
export { BrokerAlertPipeline } from "./BrokerAlertPipeline";
|
||||||
|
export { BROKER_REGISTRY, getBrokerById, getActiveBrokers } from "./brokerRegistry";
|
||||||
|
export type { PersonalInfo, RemovalJob, BrokerEntry } from "./types";
|
||||||
39
services/removebrokers/src/types.ts
Normal file
39
services/removebrokers/src/types.ts
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
import { RemovalMethod, RemovalStatus, BrokerCategory } from "@shieldai/types";
|
||||||
|
|
||||||
|
export interface PersonalInfo {
|
||||||
|
fullName: string;
|
||||||
|
email?: string;
|
||||||
|
phone?: string;
|
||||||
|
address?: {
|
||||||
|
street?: string;
|
||||||
|
city?: string;
|
||||||
|
state?: string;
|
||||||
|
zip?: string;
|
||||||
|
};
|
||||||
|
dob?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BrokerEntry {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
domain: string;
|
||||||
|
category: BrokerCategory;
|
||||||
|
removalMethod: RemovalMethod;
|
||||||
|
removalUrl?: string;
|
||||||
|
requiresAccount: boolean;
|
||||||
|
requiresVerification: boolean;
|
||||||
|
estimatedDays: number;
|
||||||
|
isActive: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RemovalJob {
|
||||||
|
requestId: string;
|
||||||
|
brokerId: string;
|
||||||
|
brokerName: string;
|
||||||
|
personalInfo: PersonalInfo;
|
||||||
|
method: RemovalMethod;
|
||||||
|
attempt: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const MAX_REMOVAL_ATTEMPTS = 3;
|
||||||
|
export const RETRY_DELAY_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||||
36
services/removebrokers/test/brokerRegistry.test.ts
Normal file
36
services/removebrokers/test/brokerRegistry.test.ts
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
import { BROKER_REGISTRY, getBrokerById, getActiveBrokers } from "../src/brokerRegistry";
|
||||||
|
import { RemovalMethod } from "@shieldai/types";
|
||||||
|
|
||||||
|
describe("BrokerRegistry", () => {
|
||||||
|
it("should have brokers registered", () => {
|
||||||
|
expect(BROKER_REGISTRY.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should find broker by id", () => {
|
||||||
|
const broker = getBrokerById("whitepages");
|
||||||
|
expect(broker).toBeDefined();
|
||||||
|
expect(broker?.name).toBe("Whitepages");
|
||||||
|
expect(broker?.domain).toBe("whitepages.com");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should return undefined for unknown broker", () => {
|
||||||
|
const broker = getBrokerById("nonexistent");
|
||||||
|
expect(broker).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should return only active brokers", () => {
|
||||||
|
const active = getActiveBrokers();
|
||||||
|
expect(active.length).toBeGreaterThan(0);
|
||||||
|
for (const broker of active) {
|
||||||
|
expect(broker.isActive).toBe(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should have varied removal methods", () => {
|
||||||
|
const methods = new Set(BROKER_REGISTRY.map((b) => b.removalMethod));
|
||||||
|
expect(methods.has(RemovalMethod.AUTOMATED)).toBe(true);
|
||||||
|
expect(methods.has(RemovalMethod.MANUAL_FORM)).toBe(true);
|
||||||
|
expect(methods.has(RemovalMethod.EMAIL)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
11
services/removebrokers/tsconfig.json
Normal file
11
services/removebrokers/tsconfig.json
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"outDir": "./dist",
|
||||||
|
"rootDir": "./src",
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"module": "ES2022",
|
||||||
|
"moduleResolution": "Bundler"
|
||||||
|
},
|
||||||
|
"include": ["src/**/*.ts"]
|
||||||
|
}
|
||||||
14
services/removebrokers/vitest.config.ts
Normal file
14
services/removebrokers/vitest.config.ts
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
import { defineConfig } from 'vitest/config';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
test: {
|
||||||
|
globals: true,
|
||||||
|
environment: 'node',
|
||||||
|
include: ['test/**/*.test.ts'],
|
||||||
|
coverage: {
|
||||||
|
provider: 'v8',
|
||||||
|
include: ['src/**/*.ts'],
|
||||||
|
exclude: ['src/index.ts'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user