Add Info Broker Removal service (FRE-5402)
New service for helping clients remove personal listings from data broker sites. Service features: - BrokerRegistry: Catalog of 20+ data brokers with removal methods - RemoveBrokersService: Core service for scanning, creating removal requests, submitting removals, and verifying completions - RemoveBrokersScheduler: Automated processing of pending removals and verification of completed removals - BrokerAlertPipeline: Alert integration for listing discoveries and removal status API endpoints (/removebrokers): - GET /brokers - List available data brokers - GET /status - Get removal request status and stats - POST /scan - Scan for personal listings across brokers - POST /request - Create a new removal request - GET /request/:id - Get specific removal request details - DELETE /request/:id - Cancel a removal request - POST /process - Trigger processing of pending removals - POST /verify/:id - Manually verify a removal completion DB models: InfoBroker, RemovalRequest, BrokerListing Types: BrokerStatus, RemovalStatus, RemovalMethod, and related interfaces
This commit is contained in:
Founding Engineer
25
services/removebrokers/package.json
Normal file
25
services/removebrokers/package.json
Normal file
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "@shieldai/removebrokers",
|
||||
"version": "0.1.0",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.js",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "vitest run",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"lint": "eslint src/"
|
||||
},
|
||||
"dependencies": {
|
||||
"@shieldai/db": "workspace:*",
|
||||
"@shieldai/types": "workspace:*",
|
||||
"@shieldai/shared-notifications": "workspace:*",
|
||||
"node-cache": "^5.1.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"vitest": "^4.1.5",
|
||||
"@vitest/coverage-v8": "^4.1.5"
|
||||
},
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
}
|
||||
}
|
||||
69
services/removebrokers/src/BrokerAlertPipeline.ts
Normal file
69
services/removebrokers/src/BrokerAlertPipeline.ts
Normal file
@@ -0,0 +1,69 @@
|
||||
import { AlertSource, AlertCategory, Severity, EntityType } from "@shieldai/types";
|
||||
|
||||
export interface BrokerAlertInput {
|
||||
userId: string;
|
||||
brokerName: string;
|
||||
brokerId: string;
|
||||
category: AlertCategory;
|
||||
severity: Severity;
|
||||
title: string;
|
||||
description: string;
|
||||
entities: Array<{ type: EntityType; value: string }>;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export class BrokerAlertPipeline {
|
||||
async sendListingFoundAlert(input: BrokerAlertInput) {
|
||||
const alert = {
|
||||
source: AlertSource.INFO_BROKER,
|
||||
category: AlertCategory.INFO_BROKER_LISTING,
|
||||
severity: input.severity,
|
||||
userId: input.userId,
|
||||
title: input.title,
|
||||
description: input.description,
|
||||
entities: input.entities,
|
||||
sourceAlertId: `broker_listing_${input.brokerId}_${Date.now()}`,
|
||||
payload: {
|
||||
brokerId: input.brokerId,
|
||||
brokerName: input.brokerName,
|
||||
...input.metadata,
|
||||
},
|
||||
timestamp: new Date(),
|
||||
};
|
||||
|
||||
return this.normalizeAndSend(alert);
|
||||
}
|
||||
|
||||
async sendRemovalStatusAlert(input: BrokerAlertInput) {
|
||||
const alert = {
|
||||
source: AlertSource.INFO_BROKER,
|
||||
category: AlertCategory.INFO_BROKER_REMOVAL,
|
||||
severity: input.severity,
|
||||
userId: input.userId,
|
||||
title: input.title,
|
||||
description: input.description,
|
||||
entities: input.entities,
|
||||
sourceAlertId: `broker_removal_${input.brokerId}_${Date.now()}`,
|
||||
payload: {
|
||||
brokerId: input.brokerId,
|
||||
brokerName: input.brokerName,
|
||||
...input.metadata,
|
||||
},
|
||||
timestamp: new Date(),
|
||||
};
|
||||
|
||||
return this.normalizeAndSend(alert);
|
||||
}
|
||||
|
||||
private async normalizeAndSend(alert: any) {
|
||||
try {
|
||||
const { correlationPipeline } = await import("@shieldai/correlation");
|
||||
return correlationPipeline.normalizeAlert(alert);
|
||||
} catch {
|
||||
console.error("[BrokerAlert] Failed to send alert:", alert.sourceAlertId);
|
||||
return alert;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const brokerAlertPipeline = new BrokerAlertPipeline();
|
||||
145
services/removebrokers/src/RemoveBrokersScheduler.ts
Normal file
145
services/removebrokers/src/RemoveBrokersScheduler.ts
Normal file
@@ -0,0 +1,145 @@
|
||||
import prisma from "@shieldai/db";
|
||||
import { RemovalStatus } from "@shieldai/types";
|
||||
import { removeBrokersService } from "./RemoveBrokersService";
|
||||
|
||||
const DEFAULT_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
const VERIFICATION_DELAY_MS = 7 * 24 * 60 * 60 * 1000; // 7 days after submission
|
||||
|
||||
export class RemoveBrokersScheduler {
|
||||
private intervalId: NodeJS.Timeout | null = null;
|
||||
private intervalMs: number;
|
||||
|
||||
constructor(intervalMs: number = DEFAULT_SCAN_INTERVAL_MS) {
|
||||
this.intervalMs = intervalMs;
|
||||
}
|
||||
|
||||
isRunning(): boolean {
|
||||
return this.intervalId !== null;
|
||||
}
|
||||
|
||||
start() {
|
||||
if (this.isRunning()) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.intervalId = setInterval(async () => {
|
||||
try {
|
||||
await this.runCycle();
|
||||
} catch (error) {
|
||||
console.error("[RemoveBrokers] Scheduler cycle error:", error);
|
||||
}
|
||||
}, this.intervalMs);
|
||||
|
||||
console.log("[RemoveBrokers] Scheduler started");
|
||||
}
|
||||
|
||||
stop() {
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
console.log("[RemoveBrokers] Scheduler stopped");
|
||||
}
|
||||
}
|
||||
|
||||
async runCycle() {
|
||||
console.log("[RemoveBrokers] Running scheduler cycle...");
|
||||
|
||||
const results = {
|
||||
processed: 0,
|
||||
verified: 0,
|
||||
errors: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
const processResults = await removeBrokersService.processPendingRequests();
|
||||
results.processed = processResults.length;
|
||||
} catch (error) {
|
||||
console.error("[RemoveBrokers] Process pending error:", error);
|
||||
results.errors++;
|
||||
}
|
||||
|
||||
try {
|
||||
const verifyResults = await this.verifyCompletedRemovals();
|
||||
results.verified = verifyResults.length;
|
||||
} catch (error) {
|
||||
console.error("[RemoveBrokers] Verify completions error:", error);
|
||||
results.errors++;
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[RemoveBrokers] Cycle complete: ${JSON.stringify(results)}`,
|
||||
);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async runScan(subscriptionId: string) {
|
||||
const subscription = await prisma.subscription.findUnique({
|
||||
where: { id: subscriptionId },
|
||||
});
|
||||
|
||||
if (!subscription) {
|
||||
throw new Error("Subscription not found");
|
||||
}
|
||||
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: subscription.userId },
|
||||
select: { name: true, email: true },
|
||||
});
|
||||
|
||||
if (!user?.name) {
|
||||
throw new Error("User name required for scanning");
|
||||
}
|
||||
|
||||
const personalInfo = {
|
||||
fullName: user.name,
|
||||
email: user.email,
|
||||
};
|
||||
|
||||
const results = await removeBrokersService.scanForListings(
|
||||
subscriptionId,
|
||||
personalInfo,
|
||||
);
|
||||
|
||||
return {
|
||||
subscriptionId,
|
||||
brokersScanned: results.length,
|
||||
listingsFound: results.filter((r) => r.found).length,
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
private async verifyCompletedRemovals() {
|
||||
const submitted = await prisma.removalRequest.findMany({
|
||||
where: {
|
||||
status: RemovalStatus.SUBMITTED,
|
||||
submittedAt: {
|
||||
lte: new Date(Date.now() - VERIFICATION_DELAY_MS),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const results = [];
|
||||
|
||||
for (const request of submitted) {
|
||||
try {
|
||||
const verification = await removeBrokersService.verifyRemoval(
|
||||
request.id,
|
||||
);
|
||||
results.push({
|
||||
requestId: request.id,
|
||||
...verification,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[RemoveBrokers] Verification error for ${request.id}:`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
export const removeBrokersScheduler = new RemoveBrokersScheduler();
|
||||
478
services/removebrokers/src/RemoveBrokersService.ts
Normal file
478
services/removebrokers/src/RemoveBrokersService.ts
Normal file
@@ -0,0 +1,478 @@
|
||||
import prisma from "@shieldai/db";
|
||||
import { RemovalStatus, RemovalMethod } from "@shieldai/types";
|
||||
import { getBrokerById, getActiveBrokers } from "./brokerRegistry";
|
||||
import type { PersonalInfo, RemovalJob, BrokerEntry } from "./types";
|
||||
import { MAX_REMOVAL_ATTEMPTS, RETRY_DELAY_MS } from "./types";
|
||||
|
||||
export class RemoveBrokersService {
|
||||
async scanForListings(subscriptionId: string, personalInfo: PersonalInfo) {
|
||||
const brokers = getActiveBrokers();
|
||||
const results = [];
|
||||
|
||||
for (const broker of brokers) {
|
||||
const existingListing = await prisma.brokerListing.findFirst({
|
||||
where: {
|
||||
subscriptionId,
|
||||
brokerId: broker.id,
|
||||
isRemoved: false,
|
||||
},
|
||||
});
|
||||
|
||||
if (existingListing) {
|
||||
results.push({
|
||||
brokerId: broker.id,
|
||||
brokerName: broker.name,
|
||||
found: true,
|
||||
listingId: existingListing.id,
|
||||
url: existingListing.url,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const found = await this.checkBrokerListing(broker, personalInfo);
|
||||
if (found) {
|
||||
const listing = await prisma.brokerListing.create({
|
||||
data: {
|
||||
subscriptionId,
|
||||
brokerId: broker.id,
|
||||
url: found.url,
|
||||
dataFound: found.dataFound,
|
||||
isRemoved: false,
|
||||
},
|
||||
});
|
||||
|
||||
results.push({
|
||||
brokerId: broker.id,
|
||||
brokerName: broker.name,
|
||||
found: true,
|
||||
listingId: listing.id,
|
||||
url: found.url,
|
||||
});
|
||||
} else {
|
||||
results.push({
|
||||
brokerId: broker.id,
|
||||
brokerName: broker.name,
|
||||
found: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async createRemovalRequest(
|
||||
subscriptionId: string,
|
||||
brokerId: string,
|
||||
personalInfo: PersonalInfo,
|
||||
notes?: string,
|
||||
) {
|
||||
const broker = getBrokerById(brokerId);
|
||||
if (!broker) {
|
||||
throw new Error(`Broker not found: ${brokerId}`);
|
||||
}
|
||||
|
||||
const existing = await prisma.removalRequest.findFirst({
|
||||
where: {
|
||||
subscriptionId,
|
||||
brokerId,
|
||||
status: { in: [RemovalStatus.PENDING, RemovalStatus.SUBMITTED, RemovalStatus.IN_PROGRESS] },
|
||||
},
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
throw new Error(`Active removal request already exists for ${broker.name}`);
|
||||
}
|
||||
|
||||
const request = await prisma.removalRequest.create({
|
||||
data: {
|
||||
subscriptionId,
|
||||
brokerId,
|
||||
status: RemovalStatus.PENDING,
|
||||
personalInfo: personalInfo as any,
|
||||
method: broker.removalMethod,
|
||||
notes,
|
||||
},
|
||||
});
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
async submitRemoval(job: RemovalJob): Promise<boolean> {
|
||||
const broker = getBrokerById(job.brokerId);
|
||||
if (!broker) {
|
||||
throw new Error(`Broker not found: ${job.brokerId}`);
|
||||
}
|
||||
|
||||
switch (job.method) {
|
||||
case RemovalMethod.AUTOMATED:
|
||||
return await this.submitAutomatedRemoval(job, broker);
|
||||
case RemovalMethod.MANUAL_FORM:
|
||||
return await this.submitManualFormRemoval(job, broker);
|
||||
case RemovalMethod.EMAIL:
|
||||
return await this.submitEmailRemoval(job, broker);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async processPendingRequests() {
|
||||
const pending = await prisma.removalRequest.findMany({
|
||||
where: {
|
||||
status: RemovalStatus.PENDING,
|
||||
OR: [
|
||||
{ nextRetryAt: null },
|
||||
{ nextRetryAt: { lte: new Date() } },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const results = [];
|
||||
|
||||
for (const request of pending) {
|
||||
try {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: { status: RemovalStatus.IN_PROGRESS },
|
||||
});
|
||||
|
||||
const job: RemovalJob = {
|
||||
requestId: request.id,
|
||||
brokerId: request.brokerId,
|
||||
brokerName: request.brokerId,
|
||||
personalInfo: request.personalInfo as PersonalInfo,
|
||||
method: request.method,
|
||||
attempt: request.attempts + 1,
|
||||
};
|
||||
|
||||
const success = await this.submitRemoval(job);
|
||||
|
||||
if (success) {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: {
|
||||
status: RemovalStatus.SUBMITTED,
|
||||
attempts: request.attempts + 1,
|
||||
submittedAt: new Date(),
|
||||
},
|
||||
});
|
||||
results.push({ requestId: request.id, status: "submitted" });
|
||||
} else if (request.attempts + 1 >= MAX_REMOVAL_ATTEMPTS) {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: {
|
||||
status: RemovalStatus.FAILED,
|
||||
attempts: request.attempts + 1,
|
||||
error: "Max attempts reached",
|
||||
},
|
||||
});
|
||||
results.push({ requestId: request.id, status: "failed" });
|
||||
} else {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: {
|
||||
attempts: request.attempts + 1,
|
||||
nextRetryAt: new Date(Date.now() + RETRY_DELAY_MS),
|
||||
},
|
||||
});
|
||||
results.push({ requestId: request.id, status: "retry_scheduled" });
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: {
|
||||
status: RemovalStatus.PENDING,
|
||||
error: message,
|
||||
nextRetryAt: new Date(Date.now() + RETRY_DELAY_MS),
|
||||
},
|
||||
});
|
||||
results.push({ requestId: request.id, status: "error", error: message });
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async verifyRemoval(requestId: string) {
|
||||
const request = await prisma.removalRequest.findUnique({
|
||||
where: { id: requestId },
|
||||
include: { broker: true },
|
||||
});
|
||||
|
||||
if (!request) {
|
||||
throw new Error(`Removal request not found: ${requestId}`);
|
||||
}
|
||||
|
||||
const personalInfo = request.personalInfo as PersonalInfo;
|
||||
const stillListed = await this.checkBrokerListing(
|
||||
request.broker,
|
||||
personalInfo,
|
||||
);
|
||||
|
||||
if (!stillListed) {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: requestId },
|
||||
data: {
|
||||
status: RemovalStatus.COMPLETED,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
await prisma.brokerListing.updateMany({
|
||||
where: {
|
||||
removalRequestId: requestId,
|
||||
isRemoved: false,
|
||||
},
|
||||
data: {
|
||||
isRemoved: true,
|
||||
removedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
return { completed: true };
|
||||
}
|
||||
|
||||
return { completed: false, stillListed: true };
|
||||
}
|
||||
|
||||
async getRemovalStatus(subscriptionId: string) {
|
||||
const requests = await prisma.removalRequest.findMany({
|
||||
where: { subscriptionId },
|
||||
include: { broker: true },
|
||||
orderBy: { updatedAt: "desc" },
|
||||
});
|
||||
|
||||
return requests.map((r: any) => ({
|
||||
id: r.id,
|
||||
brokerId: r.brokerId,
|
||||
brokerName: r.broker.name,
|
||||
status: r.status,
|
||||
method: r.method,
|
||||
attempts: r.attempts,
|
||||
submittedAt: r.submittedAt,
|
||||
completedAt: r.completedAt,
|
||||
error: r.error,
|
||||
createdAt: r.createdAt,
|
||||
updatedAt: r.updatedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
async getAvailableBrokers(): Promise<BrokerEntry[]> {
|
||||
return getActiveBrokers();
|
||||
}
|
||||
|
||||
// ---- Private methods ----
|
||||
|
||||
private async checkBrokerListing(
|
||||
broker: BrokerEntry,
|
||||
personalInfo: PersonalInfo,
|
||||
) {
|
||||
const searchUrl = this.buildSearchUrl(broker, personalInfo);
|
||||
try {
|
||||
const response = await fetch(searchUrl, {
|
||||
headers: {
|
||||
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||
},
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const listingUrl = this.extractListingUrl(html, searchUrl);
|
||||
|
||||
if (!listingUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const dataFound = this.extractPersonalData(html, personalInfo);
|
||||
|
||||
return {
|
||||
url: listingUrl,
|
||||
dataFound,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private buildSearchUrl(broker: BrokerEntry, info: PersonalInfo): string {
|
||||
const nameParts = info.fullName.split(" ");
|
||||
const firstName = nameParts[0] || "";
|
||||
const lastName = nameParts.slice(1).join(" ") || "";
|
||||
|
||||
const urlMap: Record<string, (f: string, l: string, s: string) => string> = {
|
||||
whitepages: (f, l) =>
|
||||
`https://www.whitepages.com/people/${f.toLowerCase()}-${l.toLowerCase()}`,
|
||||
spokeo: (f, l) =>
|
||||
`https://www.spokeo.com/search?q=${encodeURIComponent(info.fullName)}`,
|
||||
truepeoplesearch: (f, l) =>
|
||||
`https://www.truepeoplesearch.com/name/${encodeURIComponent(info.fullName)}`,
|
||||
peoplefinders: (f, l) =>
|
||||
`https://www.peoplefinders.com/results?name=${encodeURIComponent(info.fullName)}`,
|
||||
thatsmth: (f, l) =>
|
||||
`https://thatsmth.com/name/${encodeURIComponent(info.fullName)}`,
|
||||
fastpeoplesearch: (f, l) =>
|
||||
`https://www.fastpeoplesearch.com/name/${encodeURIComponent(info.fullName)}`,
|
||||
};
|
||||
|
||||
const builder = urlMap[broker.id];
|
||||
if (builder) {
|
||||
return builder(firstName, lastName, info.address?.state || "");
|
||||
}
|
||||
|
||||
return `https://${broker.domain}/search?q=${encodeURIComponent(info.fullName)}`;
|
||||
}
|
||||
|
||||
private extractListingUrl(html: string, searchUrl: string): string | null {
|
||||
const profilePatterns = [
|
||||
/href="([^"]*\/people\/[^"]+)"/,
|
||||
/href="([^"]*\/profile\/[^"]+)"/,
|
||||
/href="([^"]*\/results\/[^"]+)"/,
|
||||
];
|
||||
|
||||
for (const pattern of profilePatterns) {
|
||||
const match = html.match(pattern);
|
||||
if (match) {
|
||||
let url = match[1];
|
||||
if (url.startsWith("/")) {
|
||||
const urlObj = new URL(searchUrl);
|
||||
url = `${urlObj.protocol}//${urlObj.host}${url}`;
|
||||
}
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private extractPersonalData(html: string, _info: PersonalInfo): Record<string, string> {
|
||||
const data: Record<string, string> = {};
|
||||
|
||||
const phonePattern = /\b(\d{3}[-.]?\d{3}[-.]?\d{4})\b/;
|
||||
const phoneMatch = html.match(phonePattern);
|
||||
if (phoneMatch) {
|
||||
data.phoneNumber = phoneMatch[1];
|
||||
}
|
||||
|
||||
const addressPattern = /(\d+\s+[A-Za-z\s]+,\s*[A-Z]{2}\s*\d{5})/;
|
||||
const addressMatch = html.match(addressPattern);
|
||||
if (addressMatch) {
|
||||
data.address = addressMatch[1];
|
||||
}
|
||||
|
||||
const emailPattern = /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/;
|
||||
const emailMatch = html.match(emailPattern);
|
||||
if (emailMatch) {
|
||||
data.email = emailMatch[1];
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private async submitAutomatedRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||
if (!broker.removalUrl) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(broker.removalUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||
},
|
||||
body: new URLSearchParams({
|
||||
full_name: job.personalInfo.fullName,
|
||||
email: job.personalInfo.email || "",
|
||||
phone: job.personalInfo.phone || "",
|
||||
address: job.personalInfo.address
|
||||
? `${job.personalInfo.address.street || ""}, ${job.personalInfo.address.city || ""}, ${job.personalInfo.address.state || ""} ${job.personalInfo.address.zip || ""}`.trim()
|
||||
: "",
|
||||
dob: job.personalInfo.dob || "",
|
||||
}),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
return response.ok || response.status === 302;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async submitManualFormRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||
if (!broker.removalUrl) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(broker.removalUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: job.personalInfo.fullName,
|
||||
email: job.personalInfo.email,
|
||||
phone: job.personalInfo.phone,
|
||||
address: job.personalInfo.address,
|
||||
reason: "privacy_removal",
|
||||
}),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
return response.ok || response.status === 302;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async submitEmailRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||
const emailBody = this.generateEmailRemovalRequest(job, broker);
|
||||
|
||||
try {
|
||||
const emailServiceUrl = process.env.REMOVAL_EMAIL_SERVICE_URL;
|
||||
if (!emailServiceUrl) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const response = await fetch(emailServiceUrl, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
to: `privacy@${broker.domain}`,
|
||||
subject: `Data Removal Request - ${job.personalInfo.fullName}`,
|
||||
body: emailBody,
|
||||
}),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private generateEmailRemovalRequest(job: RemovalJob, broker: BrokerEntry): string {
|
||||
return `Dear ${broker.name} Privacy Team,
|
||||
|
||||
I am requesting the removal of my personal information from your website (${broker.domain}).
|
||||
|
||||
My Details:
|
||||
- Full Name: ${job.personalInfo.fullName}
|
||||
${job.personalInfo.email ? `- Email: ${job.personalInfo.email}` : ""}
|
||||
${job.personalInfo.phone ? `- Phone: ${job.personalInfo.phone}` : ""}
|
||||
${job.personalInfo.address ? `- Address: ${job.personalInfo.address.street || ""}, ${job.personalInfo.address.city || ""}, ${job.personalInfo.address.state || ""} ${job.personalInfo.address.zip || ""}`.trim() : ""}
|
||||
${job.personalInfo.dob ? `- Date of Birth: ${job.personalInfo.dob}` : ""}
|
||||
|
||||
I do not consent to the publication of my personal information on your site. Please process this removal request within 30 days as required by applicable privacy laws.
|
||||
|
||||
Thank you,
|
||||
${job.personalInfo.fullName}`;
|
||||
}
|
||||
}
|
||||
|
||||
export const removeBrokersService = new RemoveBrokersService();
|
||||
261
services/removebrokers/src/brokerRegistry.ts
Normal file
261
services/removebrokers/src/brokerRegistry.ts
Normal file
@@ -0,0 +1,261 @@
|
||||
import { RemovalMethod } from "@shieldai/types";
|
||||
import type { BrokerEntry } from "./types";
|
||||
|
||||
export const BROKER_REGISTRY: BrokerEntry[] = [
|
||||
{
|
||||
id: "whitepages",
|
||||
name: "Whitepages",
|
||||
domain: "whitepages.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://www.whitepages.com/optout",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 7,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "spokeo",
|
||||
name: "Spokeo",
|
||||
domain: "spokeo.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://www.spokeo.com/privacy/removal-request",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "truepeoplesearch",
|
||||
name: "TruePeopleSearch",
|
||||
domain: "truepeoplesearch.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.AUTOMATED,
|
||||
removalUrl: "https://www.truepeoplesearch.com/remove-your-info",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 3,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "peoplefinders",
|
||||
name: "PeopleFinders",
|
||||
domain: "peoplefinders.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://www.peoplefinders.com/privacy-policy",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "thatsmth",
|
||||
name: "That's Them",
|
||||
domain: "thatsmth.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.AUTOMATED,
|
||||
removalUrl: "https://thatsmth.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 7,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "fastpeoplesearch",
|
||||
name: "FastPeopleSearch",
|
||||
domain: "fastpeoplesearch.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.AUTOMATED,
|
||||
removalUrl: "https://www.fastpeoplesearch.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 5,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "backgroundcheck",
|
||||
name: "BackgroundCheck",
|
||||
domain: "backgroundcheck.com",
|
||||
category: "background_check",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://www.backgroundcheck.com/removal",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "freepeopledirectory",
|
||||
name: "Free People Directory",
|
||||
domain: "freepeopledirectory.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.AUTOMATED,
|
||||
removalUrl: "https://freepeopledirectory.com/optout",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 7,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "radaris",
|
||||
name: "Radaris",
|
||||
domain: "radaris.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.EMAIL,
|
||||
removalUrl: undefined,
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 30,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "zynda",
|
||||
name: "Zynda",
|
||||
domain: "zynda.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://zynda.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "addressinator",
|
||||
name: "Addressinator",
|
||||
domain: "addressinator.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://addressinator.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "familytree Now",
|
||||
name: "FamilyTree Now",
|
||||
domain: "familytreenow.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.EMAIL,
|
||||
removalUrl: undefined,
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 30,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "accuratebackground",
|
||||
name: "Accurate Background",
|
||||
domain: "accuratebackground.com",
|
||||
category: "background_check",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://www.accuratebackground.com/optout",
|
||||
requiresAccount: true,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 30,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "instantcheckmate",
|
||||
name: "Instant Checkmate",
|
||||
domain: "instantcheckmate.com",
|
||||
category: "background_check",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://www.instantcheckmate.com/opt-out",
|
||||
requiresAccount: true,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 30,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "pthree",
|
||||
name: "P3 (People Finders)",
|
||||
domain: "pthree.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://www.pthree.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "sortedbee",
|
||||
name: "Sorted Bee",
|
||||
domain: "sortedbee.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://www.sortedbee.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "ussearch",
|
||||
name: "US Search",
|
||||
domain: "ussearch.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.AUTOMATED,
|
||||
removalUrl: "https://www.ussearch.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 7,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "tellme",
|
||||
name: "Tell me Online Info",
|
||||
domain: "tellmeonlineinfo.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.MANUAL_FORM,
|
||||
removalUrl: "https://tellmeonlineinfo.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "synpeople",
|
||||
name: "Synpeople",
|
||||
domain: "synpeople.com",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.AUTOMATED,
|
||||
removalUrl: "https://www.synpeople.com/opt-out",
|
||||
requiresAccount: false,
|
||||
requiresVerification: true,
|
||||
estimatedDays: 7,
|
||||
isActive: true,
|
||||
},
|
||||
{
|
||||
id: "atomdata",
|
||||
name: "Atom Data",
|
||||
domain: "atomdata.xyz",
|
||||
category: "people_search",
|
||||
removalMethod: RemovalMethod.EMAIL,
|
||||
removalUrl: undefined,
|
||||
requiresAccount: false,
|
||||
requiresVerification: false,
|
||||
estimatedDays: 14,
|
||||
isActive: true,
|
||||
},
|
||||
];
|
||||
|
||||
export function getBrokerById(id: string): BrokerEntry | undefined {
|
||||
return BROKER_REGISTRY.find((b) => b.id === id);
|
||||
}
|
||||
|
||||
export function getActiveBrokers(): BrokerEntry[] {
|
||||
return BROKER_REGISTRY.filter((b) => b.isActive);
|
||||
}
|
||||
|
||||
export function getBrokersByCategory(category: string): BrokerEntry[] {
|
||||
return BROKER_REGISTRY.filter((b) => b.category === category);
|
||||
}
|
||||
|
||||
export function getBrokersByMethod(method: RemovalMethod): BrokerEntry[] {
|
||||
return BROKER_REGISTRY.filter((b) => b.removalMethod === method);
|
||||
}
|
||||
8
services/removebrokers/src/index.ts
Normal file
8
services/removebrokers/src/index.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
export { removeBrokersService } from "./RemoveBrokersService";
|
||||
export { RemoveBrokersService } from "./RemoveBrokersService";
|
||||
export { removeBrokersScheduler } from "./RemoveBrokersScheduler";
|
||||
export { RemoveBrokersScheduler } from "./RemoveBrokersScheduler";
|
||||
export { brokerAlertPipeline } from "./BrokerAlertPipeline";
|
||||
export { BrokerAlertPipeline } from "./BrokerAlertPipeline";
|
||||
export { BROKER_REGISTRY, getBrokerById, getActiveBrokers } from "./brokerRegistry";
|
||||
export type { PersonalInfo, RemovalJob, BrokerEntry } from "./types";
|
||||
39
services/removebrokers/src/types.ts
Normal file
39
services/removebrokers/src/types.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { RemovalMethod, RemovalStatus, BrokerCategory } from "@shieldai/types";
|
||||
|
||||
export interface PersonalInfo {
|
||||
fullName: string;
|
||||
email?: string;
|
||||
phone?: string;
|
||||
address?: {
|
||||
street?: string;
|
||||
city?: string;
|
||||
state?: string;
|
||||
zip?: string;
|
||||
};
|
||||
dob?: string;
|
||||
}
|
||||
|
||||
export interface BrokerEntry {
|
||||
id: string;
|
||||
name: string;
|
||||
domain: string;
|
||||
category: BrokerCategory;
|
||||
removalMethod: RemovalMethod;
|
||||
removalUrl?: string;
|
||||
requiresAccount: boolean;
|
||||
requiresVerification: boolean;
|
||||
estimatedDays: number;
|
||||
isActive: boolean;
|
||||
}
|
||||
|
||||
export interface RemovalJob {
|
||||
requestId: string;
|
||||
brokerId: string;
|
||||
brokerName: string;
|
||||
personalInfo: PersonalInfo;
|
||||
method: RemovalMethod;
|
||||
attempt: number;
|
||||
}
|
||||
|
||||
export const MAX_REMOVAL_ATTEMPTS = 3;
|
||||
export const RETRY_DELAY_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
36
services/removebrokers/test/brokerRegistry.test.ts
Normal file
36
services/removebrokers/test/brokerRegistry.test.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { BROKER_REGISTRY, getBrokerById, getActiveBrokers } from "../src/brokerRegistry";
|
||||
import { RemovalMethod } from "@shieldai/types";
|
||||
|
||||
describe("BrokerRegistry", () => {
|
||||
it("should have brokers registered", () => {
|
||||
expect(BROKER_REGISTRY.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should find broker by id", () => {
|
||||
const broker = getBrokerById("whitepages");
|
||||
expect(broker).toBeDefined();
|
||||
expect(broker?.name).toBe("Whitepages");
|
||||
expect(broker?.domain).toBe("whitepages.com");
|
||||
});
|
||||
|
||||
it("should return undefined for unknown broker", () => {
|
||||
const broker = getBrokerById("nonexistent");
|
||||
expect(broker).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should return only active brokers", () => {
|
||||
const active = getActiveBrokers();
|
||||
expect(active.length).toBeGreaterThan(0);
|
||||
for (const broker of active) {
|
||||
expect(broker.isActive).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("should have varied removal methods", () => {
|
||||
const methods = new Set(BROKER_REGISTRY.map((b) => b.removalMethod));
|
||||
expect(methods.has(RemovalMethod.AUTOMATED)).toBe(true);
|
||||
expect(methods.has(RemovalMethod.MANUAL_FORM)).toBe(true);
|
||||
expect(methods.has(RemovalMethod.EMAIL)).toBe(true);
|
||||
});
|
||||
});
|
||||
11
services/removebrokers/tsconfig.json
Normal file
11
services/removebrokers/tsconfig.json
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"skipLibCheck": true,
|
||||
"module": "ES2022",
|
||||
"moduleResolution": "Bundler"
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
14
services/removebrokers/vitest.config.ts
Normal file
14
services/removebrokers/vitest.config.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
include: ['test/**/*.test.ts'],
|
||||
coverage: {
|
||||
provider: 'v8',
|
||||
include: ['src/**/*.ts'],
|
||||
exclude: ['src/index.ts'],
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user