Add Info Broker Removal service (FRE-5402)
New service for helping clients remove personal listings from data broker sites. Service features: - BrokerRegistry: Catalog of 20+ data brokers with removal methods - RemoveBrokersService: Core service for scanning, creating removal requests, submitting removals, and verifying completions - RemoveBrokersScheduler: Automated processing of pending removals and verification of completed removals - BrokerAlertPipeline: Alert integration for listing discoveries and removal status API endpoints (/removebrokers): - GET /brokers - List available data brokers - GET /status - Get removal request status and stats - POST /scan - Scan for personal listings across brokers - POST /request - Create a new removal request - GET /request/:id - Get specific removal request details - DELETE /request/:id - Cancel a removal request - POST /process - Trigger processing of pending removals - POST /verify/:id - Manually verify a removal completion DB models: InfoBroker, RemovalRequest, BrokerListing Types: BrokerStatus, RemovalStatus, RemovalMethod, and related interfaces
This commit is contained in:
Founding Engineer
478
services/removebrokers/src/RemoveBrokersService.ts
Normal file
478
services/removebrokers/src/RemoveBrokersService.ts
Normal file
@@ -0,0 +1,478 @@
|
||||
import prisma from "@shieldai/db";
|
||||
import { RemovalStatus, RemovalMethod } from "@shieldai/types";
|
||||
import { getBrokerById, getActiveBrokers } from "./brokerRegistry";
|
||||
import type { PersonalInfo, RemovalJob, BrokerEntry } from "./types";
|
||||
import { MAX_REMOVAL_ATTEMPTS, RETRY_DELAY_MS } from "./types";
|
||||
|
||||
export class RemoveBrokersService {
|
||||
async scanForListings(subscriptionId: string, personalInfo: PersonalInfo) {
|
||||
const brokers = getActiveBrokers();
|
||||
const results = [];
|
||||
|
||||
for (const broker of brokers) {
|
||||
const existingListing = await prisma.brokerListing.findFirst({
|
||||
where: {
|
||||
subscriptionId,
|
||||
brokerId: broker.id,
|
||||
isRemoved: false,
|
||||
},
|
||||
});
|
||||
|
||||
if (existingListing) {
|
||||
results.push({
|
||||
brokerId: broker.id,
|
||||
brokerName: broker.name,
|
||||
found: true,
|
||||
listingId: existingListing.id,
|
||||
url: existingListing.url,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const found = await this.checkBrokerListing(broker, personalInfo);
|
||||
if (found) {
|
||||
const listing = await prisma.brokerListing.create({
|
||||
data: {
|
||||
subscriptionId,
|
||||
brokerId: broker.id,
|
||||
url: found.url,
|
||||
dataFound: found.dataFound,
|
||||
isRemoved: false,
|
||||
},
|
||||
});
|
||||
|
||||
results.push({
|
||||
brokerId: broker.id,
|
||||
brokerName: broker.name,
|
||||
found: true,
|
||||
listingId: listing.id,
|
||||
url: found.url,
|
||||
});
|
||||
} else {
|
||||
results.push({
|
||||
brokerId: broker.id,
|
||||
brokerName: broker.name,
|
||||
found: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async createRemovalRequest(
|
||||
subscriptionId: string,
|
||||
brokerId: string,
|
||||
personalInfo: PersonalInfo,
|
||||
notes?: string,
|
||||
) {
|
||||
const broker = getBrokerById(brokerId);
|
||||
if (!broker) {
|
||||
throw new Error(`Broker not found: ${brokerId}`);
|
||||
}
|
||||
|
||||
const existing = await prisma.removalRequest.findFirst({
|
||||
where: {
|
||||
subscriptionId,
|
||||
brokerId,
|
||||
status: { in: [RemovalStatus.PENDING, RemovalStatus.SUBMITTED, RemovalStatus.IN_PROGRESS] },
|
||||
},
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
throw new Error(`Active removal request already exists for ${broker.name}`);
|
||||
}
|
||||
|
||||
const request = await prisma.removalRequest.create({
|
||||
data: {
|
||||
subscriptionId,
|
||||
brokerId,
|
||||
status: RemovalStatus.PENDING,
|
||||
personalInfo: personalInfo as any,
|
||||
method: broker.removalMethod,
|
||||
notes,
|
||||
},
|
||||
});
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
async submitRemoval(job: RemovalJob): Promise<boolean> {
|
||||
const broker = getBrokerById(job.brokerId);
|
||||
if (!broker) {
|
||||
throw new Error(`Broker not found: ${job.brokerId}`);
|
||||
}
|
||||
|
||||
switch (job.method) {
|
||||
case RemovalMethod.AUTOMATED:
|
||||
return await this.submitAutomatedRemoval(job, broker);
|
||||
case RemovalMethod.MANUAL_FORM:
|
||||
return await this.submitManualFormRemoval(job, broker);
|
||||
case RemovalMethod.EMAIL:
|
||||
return await this.submitEmailRemoval(job, broker);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async processPendingRequests() {
|
||||
const pending = await prisma.removalRequest.findMany({
|
||||
where: {
|
||||
status: RemovalStatus.PENDING,
|
||||
OR: [
|
||||
{ nextRetryAt: null },
|
||||
{ nextRetryAt: { lte: new Date() } },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const results = [];
|
||||
|
||||
for (const request of pending) {
|
||||
try {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: { status: RemovalStatus.IN_PROGRESS },
|
||||
});
|
||||
|
||||
const job: RemovalJob = {
|
||||
requestId: request.id,
|
||||
brokerId: request.brokerId,
|
||||
brokerName: request.brokerId,
|
||||
personalInfo: request.personalInfo as PersonalInfo,
|
||||
method: request.method,
|
||||
attempt: request.attempts + 1,
|
||||
};
|
||||
|
||||
const success = await this.submitRemoval(job);
|
||||
|
||||
if (success) {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: {
|
||||
status: RemovalStatus.SUBMITTED,
|
||||
attempts: request.attempts + 1,
|
||||
submittedAt: new Date(),
|
||||
},
|
||||
});
|
||||
results.push({ requestId: request.id, status: "submitted" });
|
||||
} else if (request.attempts + 1 >= MAX_REMOVAL_ATTEMPTS) {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: {
|
||||
status: RemovalStatus.FAILED,
|
||||
attempts: request.attempts + 1,
|
||||
error: "Max attempts reached",
|
||||
},
|
||||
});
|
||||
results.push({ requestId: request.id, status: "failed" });
|
||||
} else {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: {
|
||||
attempts: request.attempts + 1,
|
||||
nextRetryAt: new Date(Date.now() + RETRY_DELAY_MS),
|
||||
},
|
||||
});
|
||||
results.push({ requestId: request.id, status: "retry_scheduled" });
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: request.id },
|
||||
data: {
|
||||
status: RemovalStatus.PENDING,
|
||||
error: message,
|
||||
nextRetryAt: new Date(Date.now() + RETRY_DELAY_MS),
|
||||
},
|
||||
});
|
||||
results.push({ requestId: request.id, status: "error", error: message });
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async verifyRemoval(requestId: string) {
|
||||
const request = await prisma.removalRequest.findUnique({
|
||||
where: { id: requestId },
|
||||
include: { broker: true },
|
||||
});
|
||||
|
||||
if (!request) {
|
||||
throw new Error(`Removal request not found: ${requestId}`);
|
||||
}
|
||||
|
||||
const personalInfo = request.personalInfo as PersonalInfo;
|
||||
const stillListed = await this.checkBrokerListing(
|
||||
request.broker,
|
||||
personalInfo,
|
||||
);
|
||||
|
||||
if (!stillListed) {
|
||||
await prisma.removalRequest.update({
|
||||
where: { id: requestId },
|
||||
data: {
|
||||
status: RemovalStatus.COMPLETED,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
await prisma.brokerListing.updateMany({
|
||||
where: {
|
||||
removalRequestId: requestId,
|
||||
isRemoved: false,
|
||||
},
|
||||
data: {
|
||||
isRemoved: true,
|
||||
removedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
return { completed: true };
|
||||
}
|
||||
|
||||
return { completed: false, stillListed: true };
|
||||
}
|
||||
|
||||
async getRemovalStatus(subscriptionId: string) {
|
||||
const requests = await prisma.removalRequest.findMany({
|
||||
where: { subscriptionId },
|
||||
include: { broker: true },
|
||||
orderBy: { updatedAt: "desc" },
|
||||
});
|
||||
|
||||
return requests.map((r: any) => ({
|
||||
id: r.id,
|
||||
brokerId: r.brokerId,
|
||||
brokerName: r.broker.name,
|
||||
status: r.status,
|
||||
method: r.method,
|
||||
attempts: r.attempts,
|
||||
submittedAt: r.submittedAt,
|
||||
completedAt: r.completedAt,
|
||||
error: r.error,
|
||||
createdAt: r.createdAt,
|
||||
updatedAt: r.updatedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
async getAvailableBrokers(): Promise<BrokerEntry[]> {
|
||||
return getActiveBrokers();
|
||||
}
|
||||
|
||||
// ---- Private methods ----
|
||||
|
||||
private async checkBrokerListing(
|
||||
broker: BrokerEntry,
|
||||
personalInfo: PersonalInfo,
|
||||
) {
|
||||
const searchUrl = this.buildSearchUrl(broker, personalInfo);
|
||||
try {
|
||||
const response = await fetch(searchUrl, {
|
||||
headers: {
|
||||
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||
},
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const listingUrl = this.extractListingUrl(html, searchUrl);
|
||||
|
||||
if (!listingUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const dataFound = this.extractPersonalData(html, personalInfo);
|
||||
|
||||
return {
|
||||
url: listingUrl,
|
||||
dataFound,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private buildSearchUrl(broker: BrokerEntry, info: PersonalInfo): string {
|
||||
const nameParts = info.fullName.split(" ");
|
||||
const firstName = nameParts[0] || "";
|
||||
const lastName = nameParts.slice(1).join(" ") || "";
|
||||
|
||||
const urlMap: Record<string, (f: string, l: string, s: string) => string> = {
|
||||
whitepages: (f, l) =>
|
||||
`https://www.whitepages.com/people/${f.toLowerCase()}-${l.toLowerCase()}`,
|
||||
spokeo: (f, l) =>
|
||||
`https://www.spokeo.com/search?q=${encodeURIComponent(info.fullName)}`,
|
||||
truepeoplesearch: (f, l) =>
|
||||
`https://www.truepeoplesearch.com/name/${encodeURIComponent(info.fullName)}`,
|
||||
peoplefinders: (f, l) =>
|
||||
`https://www.peoplefinders.com/results?name=${encodeURIComponent(info.fullName)}`,
|
||||
thatsmth: (f, l) =>
|
||||
`https://thatsmth.com/name/${encodeURIComponent(info.fullName)}`,
|
||||
fastpeoplesearch: (f, l) =>
|
||||
`https://www.fastpeoplesearch.com/name/${encodeURIComponent(info.fullName)}`,
|
||||
};
|
||||
|
||||
const builder = urlMap[broker.id];
|
||||
if (builder) {
|
||||
return builder(firstName, lastName, info.address?.state || "");
|
||||
}
|
||||
|
||||
return `https://${broker.domain}/search?q=${encodeURIComponent(info.fullName)}`;
|
||||
}
|
||||
|
||||
private extractListingUrl(html: string, searchUrl: string): string | null {
|
||||
const profilePatterns = [
|
||||
/href="([^"]*\/people\/[^"]+)"/,
|
||||
/href="([^"]*\/profile\/[^"]+)"/,
|
||||
/href="([^"]*\/results\/[^"]+)"/,
|
||||
];
|
||||
|
||||
for (const pattern of profilePatterns) {
|
||||
const match = html.match(pattern);
|
||||
if (match) {
|
||||
let url = match[1];
|
||||
if (url.startsWith("/")) {
|
||||
const urlObj = new URL(searchUrl);
|
||||
url = `${urlObj.protocol}//${urlObj.host}${url}`;
|
||||
}
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private extractPersonalData(html: string, _info: PersonalInfo): Record<string, string> {
|
||||
const data: Record<string, string> = {};
|
||||
|
||||
const phonePattern = /\b(\d{3}[-.]?\d{3}[-.]?\d{4})\b/;
|
||||
const phoneMatch = html.match(phonePattern);
|
||||
if (phoneMatch) {
|
||||
data.phoneNumber = phoneMatch[1];
|
||||
}
|
||||
|
||||
const addressPattern = /(\d+\s+[A-Za-z\s]+,\s*[A-Z]{2}\s*\d{5})/;
|
||||
const addressMatch = html.match(addressPattern);
|
||||
if (addressMatch) {
|
||||
data.address = addressMatch[1];
|
||||
}
|
||||
|
||||
const emailPattern = /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/;
|
||||
const emailMatch = html.match(emailPattern);
|
||||
if (emailMatch) {
|
||||
data.email = emailMatch[1];
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private async submitAutomatedRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||
if (!broker.removalUrl) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(broker.removalUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||
},
|
||||
body: new URLSearchParams({
|
||||
full_name: job.personalInfo.fullName,
|
||||
email: job.personalInfo.email || "",
|
||||
phone: job.personalInfo.phone || "",
|
||||
address: job.personalInfo.address
|
||||
? `${job.personalInfo.address.street || ""}, ${job.personalInfo.address.city || ""}, ${job.personalInfo.address.state || ""} ${job.personalInfo.address.zip || ""}`.trim()
|
||||
: "",
|
||||
dob: job.personalInfo.dob || "",
|
||||
}),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
return response.ok || response.status === 302;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async submitManualFormRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||
if (!broker.removalUrl) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(broker.removalUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "ShieldAI-RemoveBrokers/1.0",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: job.personalInfo.fullName,
|
||||
email: job.personalInfo.email,
|
||||
phone: job.personalInfo.phone,
|
||||
address: job.personalInfo.address,
|
||||
reason: "privacy_removal",
|
||||
}),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
return response.ok || response.status === 302;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async submitEmailRemoval(job: RemovalJob, broker: BrokerEntry): Promise<boolean> {
|
||||
const emailBody = this.generateEmailRemovalRequest(job, broker);
|
||||
|
||||
try {
|
||||
const emailServiceUrl = process.env.REMOVAL_EMAIL_SERVICE_URL;
|
||||
if (!emailServiceUrl) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const response = await fetch(emailServiceUrl, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
to: `privacy@${broker.domain}`,
|
||||
subject: `Data Removal Request - ${job.personalInfo.fullName}`,
|
||||
body: emailBody,
|
||||
}),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private generateEmailRemovalRequest(job: RemovalJob, broker: BrokerEntry): string {
|
||||
return `Dear ${broker.name} Privacy Team,
|
||||
|
||||
I am requesting the removal of my personal information from your website (${broker.domain}).
|
||||
|
||||
My Details:
|
||||
- Full Name: ${job.personalInfo.fullName}
|
||||
${job.personalInfo.email ? `- Email: ${job.personalInfo.email}` : ""}
|
||||
${job.personalInfo.phone ? `- Phone: ${job.personalInfo.phone}` : ""}
|
||||
${job.personalInfo.address ? `- Address: ${job.personalInfo.address.street || ""}, ${job.personalInfo.address.city || ""}, ${job.personalInfo.address.state || ""} ${job.personalInfo.address.zip || ""}`.trim() : ""}
|
||||
${job.personalInfo.dob ? `- Date of Birth: ${job.personalInfo.dob}` : ""}
|
||||
|
||||
I do not consent to the publication of my personal information on your site. Please process this removal request within 30 days as required by applicable privacy laws.
|
||||
|
||||
Thank you,
|
||||
${job.personalInfo.fullName}`;
|
||||
}
|
||||
}
|
||||
|
||||
export const removeBrokersService = new RemoveBrokersService();
|
||||
Reference in New Issue
Block a user