import prisma from "@shieldai/db"; import { RemovalStatus, RemovalMethod } from "@shieldai/types"; import { getBrokerById, getActiveBrokers } from "./brokerRegistry"; import type { PersonalInfo, RemovalJob, BrokerEntry } from "./types"; import { MAX_REMOVAL_ATTEMPTS, RETRY_DELAY_MS } from "./types"; export class RemoveBrokersService { async scanForListings(subscriptionId: string, personalInfo: PersonalInfo) { const brokers = getActiveBrokers(); const results = []; for (const broker of brokers) { const existingListing = await prisma.brokerListing.findFirst({ where: { subscriptionId, brokerId: broker.id, isRemoved: false, }, }); if (existingListing) { results.push({ brokerId: broker.id, brokerName: broker.name, found: true, listingId: existingListing.id, url: existingListing.url, }); continue; } const found = await this.checkBrokerListing(broker, personalInfo); if (found) { const listing = await prisma.brokerListing.create({ data: { subscriptionId, brokerId: broker.id, url: found.url, dataFound: found.dataFound, isRemoved: false, }, }); results.push({ brokerId: broker.id, brokerName: broker.name, found: true, listingId: listing.id, url: found.url, }); } else { results.push({ brokerId: broker.id, brokerName: broker.name, found: false, }); } } return results; } async createRemovalRequest( subscriptionId: string, brokerId: string, personalInfo: PersonalInfo, notes?: string, ) { const broker = getBrokerById(brokerId); if (!broker) { throw new Error(`Broker not found: ${brokerId}`); } const existing = await prisma.removalRequest.findFirst({ where: { subscriptionId, brokerId, status: { in: [RemovalStatus.PENDING, RemovalStatus.SUBMITTED, RemovalStatus.IN_PROGRESS] }, }, }); if (existing) { throw new Error(`Active removal request already exists for ${broker.name}`); } const request = await prisma.removalRequest.create({ data: { subscriptionId, brokerId, status: RemovalStatus.PENDING, personalInfo: personalInfo as any, method: broker.removalMethod, notes, }, }); return request; } async submitRemoval(job: RemovalJob): Promise { const broker = getBrokerById(job.brokerId); if (!broker) { throw new Error(`Broker not found: ${job.brokerId}`); } switch (job.method) { case RemovalMethod.AUTOMATED: return await this.submitAutomatedRemoval(job, broker); case RemovalMethod.MANUAL_FORM: return await this.submitManualFormRemoval(job, broker); case RemovalMethod.EMAIL: return await this.submitEmailRemoval(job, broker); default: return false; } } async processPendingRequests() { const pending = await prisma.removalRequest.findMany({ where: { status: RemovalStatus.PENDING, OR: [ { nextRetryAt: null }, { nextRetryAt: { lte: new Date() } }, ], }, }); const results = []; for (const request of pending) { try { await prisma.removalRequest.update({ where: { id: request.id }, data: { status: RemovalStatus.IN_PROGRESS }, }); const job: RemovalJob = { requestId: request.id, brokerId: request.brokerId, brokerName: request.brokerId, personalInfo: request.personalInfo as PersonalInfo, method: request.method, attempt: request.attempts + 1, }; const success = await this.submitRemoval(job); if (success) { await prisma.removalRequest.update({ where: { id: request.id }, data: { status: RemovalStatus.SUBMITTED, attempts: request.attempts + 1, submittedAt: new Date(), }, }); results.push({ requestId: request.id, status: "submitted" }); } else if (request.attempts + 1 >= MAX_REMOVAL_ATTEMPTS) { await prisma.removalRequest.update({ where: { id: request.id }, data: { status: RemovalStatus.FAILED, attempts: request.attempts + 1, error: "Max attempts reached", }, }); results.push({ requestId: request.id, status: "failed" }); } else { await prisma.removalRequest.update({ where: { id: request.id }, data: { attempts: request.attempts + 1, nextRetryAt: new Date(Date.now() + RETRY_DELAY_MS), }, }); results.push({ requestId: request.id, status: "retry_scheduled" }); } } catch (error) { const message = error instanceof Error ? error.message : "Unknown error"; await prisma.removalRequest.update({ where: { id: request.id }, data: { status: RemovalStatus.PENDING, error: message, nextRetryAt: new Date(Date.now() + RETRY_DELAY_MS), }, }); results.push({ requestId: request.id, status: "error", error: message }); } } return results; } async verifyRemoval(requestId: string) { const request = await prisma.removalRequest.findUnique({ where: { id: requestId }, include: { broker: true }, }); if (!request) { throw new Error(`Removal request not found: ${requestId}`); } const personalInfo = request.personalInfo as PersonalInfo; const stillListed = await this.checkBrokerListing( request.broker, personalInfo, ); if (!stillListed) { await prisma.removalRequest.update({ where: { id: requestId }, data: { status: RemovalStatus.COMPLETED, completedAt: new Date(), }, }); await prisma.brokerListing.updateMany({ where: { removalRequestId: requestId, isRemoved: false, }, data: { isRemoved: true, removedAt: new Date(), }, }); return { completed: true }; } return { completed: false, stillListed: true }; } async getRemovalStatus(subscriptionId: string) { const requests = await prisma.removalRequest.findMany({ where: { subscriptionId }, include: { broker: true }, orderBy: { updatedAt: "desc" }, }); return requests.map((r: any) => ({ id: r.id, brokerId: r.brokerId, brokerName: r.broker.name, status: r.status, method: r.method, attempts: r.attempts, submittedAt: r.submittedAt, completedAt: r.completedAt, error: r.error, createdAt: r.createdAt, updatedAt: r.updatedAt, })); } async getAvailableBrokers(): Promise { return getActiveBrokers(); } // ---- Private methods ---- private async checkBrokerListing( broker: BrokerEntry, personalInfo: PersonalInfo, ) { const searchUrl = this.buildSearchUrl(broker, personalInfo); try { const response = await fetch(searchUrl, { headers: { "User-Agent": "ShieldAI-RemoveBrokers/1.0", }, signal: AbortSignal.timeout(10000), }); if (!response.ok) { return null; } const html = await response.text(); const listingUrl = this.extractListingUrl(html, searchUrl); if (!listingUrl) { return null; } const dataFound = this.extractPersonalData(html, personalInfo); return { url: listingUrl, dataFound, }; } catch { return null; } } private buildSearchUrl(broker: BrokerEntry, info: PersonalInfo): string { const nameParts = info.fullName.split(" "); const firstName = nameParts[0] || ""; const lastName = nameParts.slice(1).join(" ") || ""; const urlMap: Record string> = { whitepages: (f, l) => `https://www.whitepages.com/people/${f.toLowerCase()}-${l.toLowerCase()}`, spokeo: (f, l) => `https://www.spokeo.com/search?q=${encodeURIComponent(info.fullName)}`, truepeoplesearch: (f, l) => `https://www.truepeoplesearch.com/name/${encodeURIComponent(info.fullName)}`, peoplefinders: (f, l) => `https://www.peoplefinders.com/results?name=${encodeURIComponent(info.fullName)}`, thatsmth: (f, l) => `https://thatsmth.com/name/${encodeURIComponent(info.fullName)}`, fastpeoplesearch: (f, l) => `https://www.fastpeoplesearch.com/name/${encodeURIComponent(info.fullName)}`, }; const builder = urlMap[broker.id]; if (builder) { return builder(firstName, lastName, info.address?.state || ""); } return `https://${broker.domain}/search?q=${encodeURIComponent(info.fullName)}`; } private extractListingUrl(html: string, searchUrl: string): string | null { const profilePatterns = [ /href="([^"]*\/people\/[^"]+)"/, /href="([^"]*\/profile\/[^"]+)"/, /href="([^"]*\/results\/[^"]+)"/, ]; for (const pattern of profilePatterns) { const match = html.match(pattern); if (match) { let url = match[1]; if (url.startsWith("/")) { const urlObj = new URL(searchUrl); url = `${urlObj.protocol}//${urlObj.host}${url}`; } return url; } } return null; } private extractPersonalData(html: string, _info: PersonalInfo): Record { const data: Record = {}; const phonePattern = /\b(\d{3}[-.]?\d{3}[-.]?\d{4})\b/; const phoneMatch = html.match(phonePattern); if (phoneMatch) { data.phoneNumber = phoneMatch[1]; } const addressPattern = /(\d+\s+[A-Za-z\s]+,\s*[A-Z]{2}\s*\d{5})/; const addressMatch = html.match(addressPattern); if (addressMatch) { data.address = addressMatch[1]; } const emailPattern = /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/; const emailMatch = html.match(emailPattern); if (emailMatch) { data.email = emailMatch[1]; } return data; } private async submitAutomatedRemoval(job: RemovalJob, broker: BrokerEntry): Promise { if (!broker.removalUrl) { return false; } try { const response = await fetch(broker.removalUrl, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": "ShieldAI-RemoveBrokers/1.0", }, body: new URLSearchParams({ full_name: job.personalInfo.fullName, email: job.personalInfo.email || "", phone: job.personalInfo.phone || "", address: job.personalInfo.address ? `${job.personalInfo.address.street || ""}, ${job.personalInfo.address.city || ""}, ${job.personalInfo.address.state || ""} ${job.personalInfo.address.zip || ""}`.trim() : "", dob: job.personalInfo.dob || "", }), signal: AbortSignal.timeout(30000), }); return response.ok || response.status === 302; } catch { return false; } } private async submitManualFormRemoval(job: RemovalJob, broker: BrokerEntry): Promise { if (!broker.removalUrl) { return false; } try { const response = await fetch(broker.removalUrl, { method: "POST", headers: { "Content-Type": "application/json", "User-Agent": "ShieldAI-RemoveBrokers/1.0", }, body: JSON.stringify({ name: job.personalInfo.fullName, email: job.personalInfo.email, phone: job.personalInfo.phone, address: job.personalInfo.address, reason: "privacy_removal", }), signal: AbortSignal.timeout(30000), }); return response.ok || response.status === 302; } catch { return false; } } private async submitEmailRemoval(job: RemovalJob, broker: BrokerEntry): Promise { const emailBody = this.generateEmailRemovalRequest(job, broker); try { const emailServiceUrl = process.env.REMOVAL_EMAIL_SERVICE_URL; if (!emailServiceUrl) { return false; } const response = await fetch(emailServiceUrl, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ to: `privacy@${broker.domain}`, subject: `Data Removal Request - ${job.personalInfo.fullName}`, body: emailBody, }), signal: AbortSignal.timeout(15000), }); return response.ok; } catch { return false; } } private generateEmailRemovalRequest(job: RemovalJob, broker: BrokerEntry): string { return `Dear ${broker.name} Privacy Team, I am requesting the removal of my personal information from your website (${broker.domain}). My Details: - Full Name: ${job.personalInfo.fullName} ${job.personalInfo.email ? `- Email: ${job.personalInfo.email}` : ""} ${job.personalInfo.phone ? `- Phone: ${job.personalInfo.phone}` : ""} ${job.personalInfo.address ? `- Address: ${job.personalInfo.address.street || ""}, ${job.personalInfo.address.city || ""}, ${job.personalInfo.address.state || ""} ${job.personalInfo.address.zip || ""}`.trim() : ""} ${job.personalInfo.dob ? `- Date of Birth: ${job.personalInfo.dob}` : ""} I do not consent to the publication of my personal information on your site. Please process this removal request within 30 days as required by applicable privacy laws. Thank you, ${job.personalInfo.fullName}`; } } export const removeBrokersService = new RemoveBrokersService();