import { ConnectionFactory } from "./database"; import { v4 as uuid } from "uuid"; import type { VisitorAnalytics, AnalyticsQuery } from "~/db/types"; import { DOWNLOAD_PATH_PREFIX, summariseDownloads } from "~/lib/download-asset"; import { CACHE_CONFIG } from "~/config"; import { VISITOR_ANALYTICS_ADDED_COLUMNS, VISITOR_ANALYTICS_DDL, VISITOR_ANALYTICS_INDEXES, VISITOR_ANALYTICS_INSERT } from "./analytics-schema"; import { truncateIpAddress } from "./request-client"; import { pruneVisitorAnalytics } from "./analytics-retention"; /** * Idempotent schema bootstrap, mirroring `nook.ts`. The table had no DDL in * this repo, so a database that never had it would accept no writes at all * (and the buffer swallows its errors, so the failure would be invisible). * Against the live database this is a no-op; it only creates what is absent. */ export const analyticsSchemaBootstrap: Promise = (async () => { const conn = ConnectionFactory(); await conn.execute(VISITOR_ANALYTICS_DDL); for (const index of VISITOR_ANALYTICS_INDEXES) { await conn.execute(index); } // SQLite has no "ADD COLUMN IF NOT EXISTS", so check before altering — // covers a database restored from before these columns existed. const info = await conn.execute(`PRAGMA table_info(VisitorAnalytics)`); const present = new Set(info.rows.map((row) => String((row as any).name))); for (const column of VISITOR_ANALYTICS_ADDED_COLUMNS) { if (!present.has(column.name)) { await conn.execute(column.ddl); } } })(); // A bootstrap failure must not become an unhandled rejection on a request // path that merely imports this module. analyticsSchemaBootstrap.catch((error) => console.error("VisitorAnalytics schema bootstrap failed:", error) ); export interface AnalyticsEntry { userId?: string | null; path: string; method: string; referrer?: string | null; userAgent?: string | null; ipAddress?: string | null; country?: string | null; deviceType?: string | null; browser?: string | null; os?: string | null; utmSource?: string | null; utmMedium?: string | null; utmCampaign?: string | null; durationMs?: number | null; fcp?: number | null; lcp?: number | null; cls?: number | null; fid?: number | null; inp?: number | null; ttfb?: number | null; domLoad?: number | null; loadComplete?: number | null; /** Funnel stage for download rows ("click" | "served"); null elsewhere. */ stage?: string | null; /** Host the client addressed, e.g. "nook.freno.me" — the site dimension. */ host?: string | null; } /** * In-memory analytics buffer for batch writing * Reduces DB writes from 3,430 to ~350 (90% reduction) */ interface AnalyticsBuffer { entries: AnalyticsEntry[]; lastFlush: number; flushTimer?: NodeJS.Timeout; } const analyticsBuffer: AnalyticsBuffer = { entries: [], lastFlush: Date.now() }; /** * Flush analytics buffer to database * Writes all buffered entries in a single batch */ async function flushAnalyticsBuffer(): Promise { if (analyticsBuffer.entries.length === 0) { return; } const entriesToWrite = [...analyticsBuffer.entries]; analyticsBuffer.entries = []; analyticsBuffer.lastFlush = Date.now(); try { const conn = ConnectionFactory(); // Batch insert - more efficient than individual inserts for (const entry of entriesToWrite) { await conn.execute({ sql: VISITOR_ANALYTICS_INSERT, args: [ uuid(), entry.userId || null, entry.path, entry.method, entry.referrer || null, entry.userAgent || null, entry.ipAddress || null, entry.country || null, entry.deviceType || null, entry.browser || null, entry.os || null, entry.utmSource || null, entry.utmMedium || null, entry.utmCampaign || null, entry.durationMs || null, entry.fcp || null, entry.lcp || null, entry.cls || null, entry.fid || null, entry.inp || null, entry.ttfb || null, entry.domLoad || null, entry.loadComplete || null, entry.stage || null, entry.host || null ] }); } } catch (error) { console.error("Failed to flush analytics buffer:", error); // Don't re-throw - analytics is non-critical } } /** * Write buffered entries now instead of waiting for the batch timer. * * The buffer only writes immediately at ANALYTICS_BATCH_SIZE entries; below * that it defers to a 30-second timeout. On a long-lived server that is * fine, but on serverless the work scheduled after a response is not * guaranteed to run before the instance is recycled — so a low-volume but * high-value event (a download click) could sit in memory and never be * recorded. Callers that must not lose a row await this. * * Safe to call concurrently: the buffer is drained synchronously before the * first await, so entries are never written twice. */ export async function flushAnalytics(): Promise { await flushAnalyticsBuffer(); } /** * Schedule periodic buffer flush */ function scheduleAnalyticsFlush(): void { if (analyticsBuffer.flushTimer) { clearTimeout(analyticsBuffer.flushTimer); } analyticsBuffer.flushTimer = setTimeout(() => { flushAnalyticsBuffer().catch((err) => console.error("Analytics flush error:", err) ); }, CACHE_CONFIG.ANALYTICS_BATCH_TIMEOUT_MS); } /** * Log visitor analytics with batching * Buffers writes in memory and flushes periodically or when batch size reached * * @param entry - Analytics data to log */ export async function logVisit(entry: AnalyticsEntry): Promise { try { analyticsBuffer.entries.push(entry); if (analyticsBuffer.entries.length >= CACHE_CONFIG.ANALYTICS_BATCH_SIZE) { await flushAnalyticsBuffer(); } else { scheduleAnalyticsFlush(); } } catch (error) { console.error("Failed to buffer visitor analytics:", error, entry); } } // Ensure buffer is flushed on process exit (best effort) if (typeof process !== "undefined") { process.on("beforeExit", () => { flushAnalyticsBuffer().catch(() => {}); }); } export async function queryAnalytics( query: AnalyticsQuery ): Promise { const conn = ConnectionFactory(); let sql = "SELECT * FROM VisitorAnalytics WHERE 1=1"; const args: any[] = []; if (query.userId) { sql += " AND user_id = ?"; args.push(query.userId); } if (query.path) { sql += " AND path = ?"; args.push(query.path); } if (query.startDate) { sql += " AND created_at >= ?"; args.push( typeof query.startDate === "string" ? query.startDate : query.startDate.toISOString() ); } if (query.endDate) { sql += " AND created_at <= ?"; args.push( typeof query.endDate === "string" ? query.endDate : query.endDate.toISOString() ); } sql += " ORDER BY created_at DESC"; if (query.limit) { sql += " LIMIT ?"; args.push(query.limit); } if (query.offset) { sql += " OFFSET ?"; args.push(query.offset); } const result = await conn.execute({ sql, args }); return result.rows.map((row) => ({ id: row.id as string, user_id: row.user_id as string | null, path: row.path as string, method: row.method as string, referrer: row.referrer as string | null, user_agent: row.user_agent as string | null, ip_address: row.ip_address as string | null, country: row.country as string | null, device_type: row.device_type as string | null, browser: row.browser as string | null, os: row.os as string | null, utm_source: row.utm_source as string | null, utm_medium: row.utm_medium as string | null, utm_campaign: row.utm_campaign as string | null, duration_ms: row.duration_ms as number | null, stage: row.stage as string | null, host: row.host as string | null, created_at: row.created_at as string })); } export async function getAnalyticsSummary(days: number = 30): Promise<{ totalVisits: number; totalPageVisits: number; totalApiCalls: number; uniqueVisitors: number; uniqueUsers: number; topPages: Array<{ path: string; count: number }>; topApiCalls: Array<{ path: string; count: number }>; topReferrers: Array<{ referrer: string; count: number }>; deviceTypes: Array<{ type: string; count: number }>; browsers: Array<{ browser: string; count: number }>; downloads: { total: number; byAsset: Array<{ asset: string; count: number }>; }; topCampaigns: Array<{ source: string | null; medium: string | null; campaign: string | null; count: number; }>; }> { const conn = ConnectionFactory(); const totalVisitsResult = await conn.execute({ sql: `SELECT COUNT(*) as count FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days')`, args: [] }); const totalVisits = (totalVisitsResult.rows[0]?.count as number) || 0; const totalPageVisitsResult = await conn.execute({ sql: `SELECT COUNT(*) as count FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND path NOT LIKE '/api/%'`, args: [] }); const totalPageVisits = (totalPageVisitsResult.rows[0]?.count as number) || 0; const totalApiCallsResult = await conn.execute({ sql: `SELECT COUNT(*) as count FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND path LIKE '/api/%'`, args: [] }); const totalApiCalls = (totalApiCallsResult.rows[0]?.count as number) || 0; // Since the FRE-369 privacy floor, `ip_address` holds a truncated prefix // (/24 for v4, /56 for v6), so this counts distinct prefixes — households and // mobile exits, not machines. It was never accurate before that either: every // row held the proxy's loopback, so the count was 1. const uniqueVisitorsResult = await conn.execute({ sql: `SELECT COUNT(DISTINCT ip_address) as count FROM VisitorAnalytics WHERE ip_address IS NOT NULL AND created_at >= datetime('now', '-${days} days')`, args: [] }); const uniqueVisitors = (uniqueVisitorsResult.rows[0]?.count as number) || 0; const uniqueUsersResult = await conn.execute({ sql: `SELECT COUNT(DISTINCT user_id) as count FROM VisitorAnalytics WHERE user_id IS NOT NULL AND created_at >= datetime('now', '-${days} days')`, args: [] }); const uniqueUsers = (uniqueUsersResult.rows[0]?.count as number) || 0; const topPagesResult = await conn.execute({ sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND path NOT LIKE '/api/%' GROUP BY path ORDER BY count DESC LIMIT 10`, args: [] }); const topPages = topPagesResult.rows.map((row) => ({ path: row.path as string, count: row.count as number })); const topApiCallsResult = await conn.execute({ sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND path LIKE '/api/%' GROUP BY path ORDER BY count DESC LIMIT 10`, args: [] }); const topApiCalls = topApiCallsResult.rows.map((row) => ({ path: row.path as string, count: row.count as number })); const topReferrersResult = await conn.execute({ sql: `SELECT referrer, COUNT(*) as count FROM VisitorAnalytics WHERE referrer IS NOT NULL AND created_at >= datetime('now', '-${days} days') GROUP BY referrer ORDER BY count DESC LIMIT 10`, args: [] }); const topReferrers = topReferrersResult.rows.map((row) => ({ referrer: row.referrer as string, count: row.count as number })); const deviceTypesResult = await conn.execute({ sql: `SELECT device_type, COUNT(*) as count FROM VisitorAnalytics WHERE device_type IS NOT NULL AND created_at >= datetime('now', '-${days} days') GROUP BY device_type ORDER BY count DESC`, args: [] }); const deviceTypes = deviceTypesResult.rows.map((row) => ({ type: row.device_type as string, count: row.count as number })); const browsersResult = await conn.execute({ sql: `SELECT browser, COUNT(*) as count FROM VisitorAnalytics WHERE browser IS NOT NULL AND created_at >= datetime('now', '-${days} days') GROUP BY browser ORDER BY count DESC LIMIT 10`, args: [] }); const browsers = browsersResult.rows.map((row) => ({ browser: row.browser as string, count: row.count as number })); // Campaign attribution (§8): which channel produced the traffic. Rows with // no tags are excluded — the "untagged" bucket is a separate question and // would otherwise dominate the list. const topCampaignsResult = await conn.execute({ sql: `SELECT utm_source, utm_medium, utm_campaign, COUNT(*) as count FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND (utm_source IS NOT NULL OR utm_medium IS NOT NULL OR utm_campaign IS NOT NULL) GROUP BY utm_source, utm_medium, utm_campaign ORDER BY count DESC LIMIT 10`, args: [] }); const topCampaigns = topCampaignsResult.rows.map((row) => ({ source: (row.utm_source as string | null) ?? null, medium: (row.utm_medium as string | null) ?? null, campaign: (row.utm_campaign as string | null) ?? null, count: row.count as number })); // Downloads: every door records one row under `/api/downloads/` via // `trackDownload`, so the asset is the path suffix and nothing has to be // parsed out of a query string. const downloadsResult = await conn.execute({ sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics WHERE path LIKE '${DOWNLOAD_PATH_PREFIX}%' AND created_at >= datetime('now', '-${days} days') GROUP BY path ORDER BY count DESC`, args: [] }); const downloads = summariseDownloads( downloadsResult.rows.map((row) => ({ path: String(row.path), count: row.count as number })) ); return { totalVisits, totalPageVisits, totalApiCalls, uniqueVisitors, uniqueUsers, topPages, topApiCalls, topReferrers, deviceTypes, browsers, topCampaigns, downloads }; } export async function getPathAnalytics( path: string, days: number = 30 ): Promise<{ totalVisits: number; uniqueVisitors: number; avgDurationMs: number | null; visitsByDay: Array<{ date: string; count: number }>; }> { const conn = ConnectionFactory(); const totalVisitsResult = await conn.execute({ sql: `SELECT COUNT(*) as count FROM VisitorAnalytics WHERE path = ? AND created_at >= datetime('now', '-${days} days')`, args: [path] }); const totalVisits = (totalVisitsResult.rows[0]?.count as number) || 0; const uniqueVisitorsResult = await conn.execute({ sql: `SELECT COUNT(DISTINCT ip_address) as count FROM VisitorAnalytics WHERE path = ? AND ip_address IS NOT NULL AND created_at >= datetime('now', '-${days} days')`, args: [path] }); const uniqueVisitors = (uniqueVisitorsResult.rows[0]?.count as number) || 0; const avgDurationResult = await conn.execute({ sql: `SELECT AVG(duration_ms) as avg FROM VisitorAnalytics WHERE path = ? AND duration_ms IS NOT NULL AND created_at >= datetime('now', '-${days} days')`, args: [path] }); const avgDurationMs = avgDurationResult.rows[0]?.avg as number | null; const visitsByDayResult = await conn.execute({ sql: `SELECT DATE(created_at) as date, COUNT(*) as count FROM VisitorAnalytics WHERE path = ? AND created_at >= datetime('now', '-${days} days') GROUP BY DATE(created_at) ORDER BY date DESC`, args: [path] }); const visitsByDay = visitsByDayResult.rows.map((row) => ({ date: row.date as string, count: row.count as number })); return { totalVisits, uniqueVisitors, avgDurationMs, visitsByDay }; } export async function getPerformanceStats(days: number = 30): Promise<{ avgLcp: number | null; avgFcp: number | null; avgCls: number | null; avgInp: number | null; avgTtfb: number | null; avgDomLoad: number | null; avgLoadComplete: number | null; p75Lcp: number | null; p75Fcp: number | null; totalWithMetrics: number; byPath: Array<{ path: string; avgLcp: number; avgFcp: number; avgCls: number; avgTtfb: number; count: number; }>; }> { const conn = ConnectionFactory(); const avgResult = await conn.execute({ sql: `SELECT AVG(lcp) as avgLcp, AVG(fcp) as avgFcp, AVG(cls) as avgCls, AVG(inp) as avgInp, AVG(ttfb) as avgTtfb, AVG(dom_load) as avgDomLoad, AVG(load_complete) as avgLoadComplete, COUNT(*) as total FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND fcp IS NOT NULL`, args: [] }); const avgRow = avgResult.rows[0] as any; // Get 75th percentile for LCP and FCP (approximation using median) const p75LcpResult = await conn.execute({ sql: `SELECT lcp as p75 FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND lcp IS NOT NULL ORDER BY lcp LIMIT 1 OFFSET ( SELECT COUNT(*) * 75 / 100 FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND lcp IS NOT NULL )`, args: [] }); const p75FcpResult = await conn.execute({ sql: `SELECT fcp as p75 FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND fcp IS NOT NULL ORDER BY fcp LIMIT 1 OFFSET ( SELECT COUNT(*) * 75 / 100 FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND fcp IS NOT NULL )`, args: [] }); const byPathResult = await conn.execute({ sql: `SELECT path, AVG(lcp) as avgLcp, AVG(fcp) as avgFcp, AVG(cls) as avgCls, AVG(ttfb) as avgTtfb, COUNT(*) as count FROM VisitorAnalytics WHERE created_at >= datetime('now', '-${days} days') AND fcp IS NOT NULL AND path NOT LIKE '/api/%' GROUP BY path ORDER BY count DESC LIMIT 20`, args: [] }); const byPath = byPathResult.rows.map((row: any) => ({ path: row.path, avgLcp: row.avgLcp || 0, avgFcp: row.avgFcp || 0, avgCls: row.avgCls || 0, avgTtfb: row.avgTtfb || 0, count: row.count })); return { avgLcp: avgRow?.avgLcp || null, avgFcp: avgRow?.avgFcp || null, avgCls: avgRow?.avgCls || null, avgInp: avgRow?.avgInp || null, avgTtfb: avgRow?.avgTtfb || null, avgDomLoad: avgRow?.avgDomLoad || null, avgLoadComplete: avgRow?.avgLoadComplete || null, p75Lcp: (p75LcpResult.rows[0] as any)?.p75 || null, p75Fcp: (p75FcpResult.rows[0] as any)?.p75 || null, totalWithMetrics: avgRow?.total || 0, byPath }; } /** * Delete visitor analytics older than `olderThanDays`. Reachable from tRPC * (admin) and from the daily retention cron (`/api/analytics/retention`); * the SQL and the input rules live in `./analytics-retention`. */ export async function cleanupOldAnalytics( olderThanDays: number ): Promise { return pruneVisitorAnalytics(ConnectionFactory(), olderThanDays); } function parseUserAgent(userAgent?: string): { deviceType: string | null; browser: string | null; os: string | null; } { if (!userAgent) { return { deviceType: null, browser: null, os: null }; } const ua = userAgent.toLowerCase(); let deviceType: string | null = "desktop"; if (ua.includes("mobile")) deviceType = "mobile"; else if (ua.includes("tablet") || ua.includes("ipad")) deviceType = "tablet"; let browser: string | null = null; if (ua.includes("edg")) browser = "edge"; else if (ua.includes("chrome")) browser = "chrome"; else if (ua.includes("firefox")) browser = "firefox"; else if (ua.includes("safari") && !ua.includes("chrome")) browser = "safari"; else if (ua.includes("opera") || ua.includes("opr")) browser = "opera"; let os: string | null = null; if (ua.includes("windows")) os = "windows"; else if (ua.includes("mac")) os = "macos"; else if (ua.includes("linux")) os = "linux"; else if (ua.includes("android")) os = "android"; else if (ua.includes("iphone") || ua.includes("ipad")) os = "ios"; return { deviceType, browser, os }; } /** * Fill in what the writer did not supply, and apply the privacy floor. * * Every analytics entry — page views, tRPC context, downloads, checkout — * passes through here, which is why the truncation lives here and not at the * call sites: a future writer cannot forget it, and no caller can persist a * full client address by accident (FRE-369, finding Q4). The audit log * (`./audit.ts`) is a separate, pre-existing surface with its own stakes and is * deliberately untouched by this change. */ export function enrichAnalyticsEntry(entry: AnalyticsEntry): AnalyticsEntry { const { deviceType, browser, os } = parseUserAgent( entry.userAgent || undefined ); return { ...entry, ipAddress: truncateIpAddress(entry.ipAddress), deviceType: entry.deviceType || deviceType, browser: entry.browser || browser, os: entry.os || os }; }