freno-dev/src/server/analytics.ts

import { ConnectionFactory } from "./database";
import { v4 as uuid } from "uuid";
import type { VisitorAnalytics, AnalyticsQuery } from "~/db/types";
import { DOWNLOAD_PATH_PREFIX, summariseDownloads } from "~/lib/download-asset";
import { CACHE_CONFIG } from "~/config";
import {
  VISITOR_ANALYTICS_ADDED_COLUMNS,
  VISITOR_ANALYTICS_DDL,
  VISITOR_ANALYTICS_INDEXES,
  VISITOR_ANALYTICS_INSERT
} from "./analytics-schema";
import { truncateIpAddress } from "./request-client";
import { pruneVisitorAnalytics } from "./analytics-retention";

/**
 * Idempotent schema bootstrap, mirroring `nook.ts`. The table had no DDL in
 * this repo, so a database that never had it would accept no writes at all
 * (and the buffer swallows its errors, so the failure would be invisible).
 * Against the live database this is a no-op; it only creates what is absent.
 */
export const analyticsSchemaBootstrap: Promise<unknown> = (async () => {
  const conn = ConnectionFactory();

  await conn.execute(VISITOR_ANALYTICS_DDL);
  for (const index of VISITOR_ANALYTICS_INDEXES) {
    await conn.execute(index);
  }

  // SQLite has no "ADD COLUMN IF NOT EXISTS", so check before altering —
  // covers a database restored from before these columns existed.
  const info = await conn.execute(`PRAGMA table_info(VisitorAnalytics)`);
  const present = new Set(info.rows.map((row) => String((row as any).name)));
  for (const column of VISITOR_ANALYTICS_ADDED_COLUMNS) {
    if (!present.has(column.name)) {
      await conn.execute(column.ddl);
    }
  }
})();

// A bootstrap failure must not become an unhandled rejection on a request
// path that merely imports this module.
analyticsSchemaBootstrap.catch((error) =>
  console.error("VisitorAnalytics schema bootstrap failed:", error)
);

export interface AnalyticsEntry {
  userId?: string | null;
  path: string;
  method: string;
  referrer?: string | null;
  userAgent?: string | null;
  ipAddress?: string | null;
  country?: string | null;
  deviceType?: string | null;
  browser?: string | null;
  os?: string | null;
  utmSource?: string | null;
  utmMedium?: string | null;
  utmCampaign?: string | null;
  durationMs?: number | null;
  fcp?: number | null;
  lcp?: number | null;
  cls?: number | null;
  fid?: number | null;
  inp?: number | null;
  ttfb?: number | null;
  domLoad?: number | null;
  loadComplete?: number | null;
  /** Funnel stage for download rows ("click" | "served"); null elsewhere. */
  stage?: string | null;
  /** Host the client addressed, e.g. "nook.freno.me" — the site dimension. */
  host?: string | null;
}

/**
 * In-memory analytics buffer for batch writing
 * Reduces DB writes from 3,430 to ~350 (90% reduction)
 */
interface AnalyticsBuffer {
  entries: AnalyticsEntry[];
  lastFlush: number;
  flushTimer?: NodeJS.Timeout;
}

const analyticsBuffer: AnalyticsBuffer = {
  entries: [],
  lastFlush: Date.now()
};

/**
 * Flush analytics buffer to database
 * Writes all buffered entries in a single batch
 */
async function flushAnalyticsBuffer(): Promise<void> {
  if (analyticsBuffer.entries.length === 0) {
    return;
  }

  const entriesToWrite = [...analyticsBuffer.entries];
  analyticsBuffer.entries = [];
  analyticsBuffer.lastFlush = Date.now();

  try {
    const conn = ConnectionFactory();

    // Batch insert - more efficient than individual inserts
    for (const entry of entriesToWrite) {
      await conn.execute({
        sql: VISITOR_ANALYTICS_INSERT,
        args: [
          uuid(),
          entry.userId || null,
          entry.path,
          entry.method,
          entry.referrer || null,
          entry.userAgent || null,
          entry.ipAddress || null,
          entry.country || null,
          entry.deviceType || null,
          entry.browser || null,
          entry.os || null,
          entry.utmSource || null,
          entry.utmMedium || null,
          entry.utmCampaign || null,
          entry.durationMs || null,
          entry.fcp || null,
          entry.lcp || null,
          entry.cls || null,
          entry.fid || null,
          entry.inp || null,
          entry.ttfb || null,
          entry.domLoad || null,
          entry.loadComplete || null,
          entry.stage || null,
          entry.host || null
        ]
      });
    }
  } catch (error) {
    console.error("Failed to flush analytics buffer:", error);
    // Don't re-throw - analytics is non-critical
  }
}

/**
 * Write buffered entries now instead of waiting for the batch timer.
 *
 * The buffer only writes immediately at ANALYTICS_BATCH_SIZE entries; below
 * that it defers to a 30-second timeout. On a long-lived server that is
 * fine, but on serverless the work scheduled after a response is not
 * guaranteed to run before the instance is recycled — so a low-volume but
 * high-value event (a download click) could sit in memory and never be
 * recorded. Callers that must not lose a row await this.
 *
 * Safe to call concurrently: the buffer is drained synchronously before the
 * first await, so entries are never written twice.
 */
export async function flushAnalytics(): Promise<void> {
  await flushAnalyticsBuffer();
}

/**
 * Schedule periodic buffer flush
 */
function scheduleAnalyticsFlush(): void {
  if (analyticsBuffer.flushTimer) {
    clearTimeout(analyticsBuffer.flushTimer);
  }

  analyticsBuffer.flushTimer = setTimeout(() => {
    flushAnalyticsBuffer().catch((err) =>
      console.error("Analytics flush error:", err)
    );
  }, CACHE_CONFIG.ANALYTICS_BATCH_TIMEOUT_MS);
}

/**
 * Log visitor analytics with batching
 * Buffers writes in memory and flushes periodically or when batch size reached
 *
 * @param entry - Analytics data to log
 */
export async function logVisit(entry: AnalyticsEntry): Promise<void> {
  try {
    analyticsBuffer.entries.push(entry);

    if (analyticsBuffer.entries.length >= CACHE_CONFIG.ANALYTICS_BATCH_SIZE) {
      await flushAnalyticsBuffer();
    } else {
      scheduleAnalyticsFlush();
    }
  } catch (error) {
    console.error("Failed to buffer visitor analytics:", error, entry);
  }
}

// Ensure buffer is flushed on process exit (best effort)
if (typeof process !== "undefined") {
  process.on("beforeExit", () => {
    flushAnalyticsBuffer().catch(() => {});
  });
}

export async function queryAnalytics(
  query: AnalyticsQuery
): Promise<VisitorAnalytics[]> {
  const conn = ConnectionFactory();

  let sql = "SELECT * FROM VisitorAnalytics WHERE 1=1";
  const args: any[] = [];

  if (query.userId) {
    sql += " AND user_id = ?";
    args.push(query.userId);
  }

  if (query.path) {
    sql += " AND path = ?";
    args.push(query.path);
  }

  if (query.startDate) {
    sql += " AND created_at >= ?";
    args.push(
      typeof query.startDate === "string"
        ? query.startDate
        : query.startDate.toISOString()
    );
  }

  if (query.endDate) {
    sql += " AND created_at <= ?";
    args.push(
      typeof query.endDate === "string"
        ? query.endDate
        : query.endDate.toISOString()
    );
  }

  sql += " ORDER BY created_at DESC";

  if (query.limit) {
    sql += " LIMIT ?";
    args.push(query.limit);
  }

  if (query.offset) {
    sql += " OFFSET ?";
    args.push(query.offset);
  }

  const result = await conn.execute({ sql, args });
  return result.rows.map((row) => ({
    id: row.id as string,
    user_id: row.user_id as string | null,
    path: row.path as string,
    method: row.method as string,
    referrer: row.referrer as string | null,
    user_agent: row.user_agent as string | null,
    ip_address: row.ip_address as string | null,
    country: row.country as string | null,
    device_type: row.device_type as string | null,
    browser: row.browser as string | null,
    os: row.os as string | null,
    utm_source: row.utm_source as string | null,
    utm_medium: row.utm_medium as string | null,
    utm_campaign: row.utm_campaign as string | null,
    duration_ms: row.duration_ms as number | null,
    stage: row.stage as string | null,
    host: row.host as string | null,
    created_at: row.created_at as string
  }));
}

export async function getAnalyticsSummary(days: number = 30): Promise<{
  totalVisits: number;
  totalPageVisits: number;
  totalApiCalls: number;
  uniqueVisitors: number;
  uniqueUsers: number;
  topPages: Array<{ path: string; count: number }>;
  topApiCalls: Array<{ path: string; count: number }>;
  topReferrers: Array<{ referrer: string; count: number }>;
  deviceTypes: Array<{ type: string; count: number }>;
  browsers: Array<{ browser: string; count: number }>;
  downloads: {
    total: number;
    byAsset: Array<{ asset: string; count: number }>;
  };
  topCampaigns: Array<{
    source: string | null;
    medium: string | null;
    campaign: string | null;
    count: number;
  }>;
}> {
  const conn = ConnectionFactory();

  const totalVisitsResult = await conn.execute({
    sql: `SELECT COUNT(*) as count FROM VisitorAnalytics 
          WHERE created_at >= datetime('now', '-${days} days')`,
    args: []
  });
  const totalVisits = (totalVisitsResult.rows[0]?.count as number) || 0;

  const totalPageVisitsResult = await conn.execute({
    sql: `SELECT COUNT(*) as count FROM VisitorAnalytics 
          WHERE created_at >= datetime('now', '-${days} days')
          AND path NOT LIKE '/api/%'`,
    args: []
  });
  const totalPageVisits = (totalPageVisitsResult.rows[0]?.count as number) || 0;

  const totalApiCallsResult = await conn.execute({
    sql: `SELECT COUNT(*) as count FROM VisitorAnalytics 
          WHERE created_at >= datetime('now', '-${days} days')
          AND path LIKE '/api/%'`,
    args: []
  });
  const totalApiCalls = (totalApiCallsResult.rows[0]?.count as number) || 0;

  // Since the FRE-369 privacy floor, `ip_address` holds a truncated prefix
  // (/24 for v4, /56 for v6), so this counts distinct prefixes — households and
  // mobile exits, not machines. It was never accurate before that either: every
  // row held the proxy's loopback, so the count was 1.
  const uniqueVisitorsResult = await conn.execute({
    sql: `SELECT COUNT(DISTINCT ip_address) as count FROM VisitorAnalytics 
          WHERE ip_address IS NOT NULL 
          AND created_at >= datetime('now', '-${days} days')`,
    args: []
  });
  const uniqueVisitors = (uniqueVisitorsResult.rows[0]?.count as number) || 0;

  const uniqueUsersResult = await conn.execute({
    sql: `SELECT COUNT(DISTINCT user_id) as count FROM VisitorAnalytics 
          WHERE user_id IS NOT NULL 
          AND created_at >= datetime('now', '-${days} days')`,
    args: []
  });
  const uniqueUsers = (uniqueUsersResult.rows[0]?.count as number) || 0;

  const topPagesResult = await conn.execute({
    sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics 
          WHERE created_at >= datetime('now', '-${days} days')
          AND path NOT LIKE '/api/%'
          GROUP BY path 
          ORDER BY count DESC 
          LIMIT 10`,
    args: []
  });
  const topPages = topPagesResult.rows.map((row) => ({
    path: row.path as string,
    count: row.count as number
  }));

  const topApiCallsResult = await conn.execute({
    sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics 
          WHERE created_at >= datetime('now', '-${days} days')
          AND path LIKE '/api/%'
          GROUP BY path 
          ORDER BY count DESC 
          LIMIT 10`,
    args: []
  });
  const topApiCalls = topApiCallsResult.rows.map((row) => ({
    path: row.path as string,
    count: row.count as number
  }));

  const topReferrersResult = await conn.execute({
    sql: `SELECT referrer, COUNT(*) as count FROM VisitorAnalytics 
          WHERE referrer IS NOT NULL 
          AND created_at >= datetime('now', '-${days} days')
          GROUP BY referrer 
          ORDER BY count DESC 
          LIMIT 10`,
    args: []
  });
  const topReferrers = topReferrersResult.rows.map((row) => ({
    referrer: row.referrer as string,
    count: row.count as number
  }));

  const deviceTypesResult = await conn.execute({
    sql: `SELECT device_type, COUNT(*) as count FROM VisitorAnalytics 
          WHERE device_type IS NOT NULL 
          AND created_at >= datetime('now', '-${days} days')
          GROUP BY device_type 
          ORDER BY count DESC`,
    args: []
  });
  const deviceTypes = deviceTypesResult.rows.map((row) => ({
    type: row.device_type as string,
    count: row.count as number
  }));

  const browsersResult = await conn.execute({
    sql: `SELECT browser, COUNT(*) as count FROM VisitorAnalytics 
          WHERE browser IS NOT NULL 
          AND created_at >= datetime('now', '-${days} days')
          GROUP BY browser 
          ORDER BY count DESC 
          LIMIT 10`,
    args: []
  });
  const browsers = browsersResult.rows.map((row) => ({
    browser: row.browser as string,
    count: row.count as number
  }));

  // Campaign attribution (§8): which channel produced the traffic. Rows with
  // no tags are excluded — the "untagged" bucket is a separate question and
  // would otherwise dominate the list.
  const topCampaignsResult = await conn.execute({
    sql: `SELECT utm_source, utm_medium, utm_campaign, COUNT(*) as count
          FROM VisitorAnalytics
          WHERE created_at >= datetime('now', '-${days} days')
          AND (utm_source IS NOT NULL OR utm_medium IS NOT NULL OR utm_campaign IS NOT NULL)
          GROUP BY utm_source, utm_medium, utm_campaign
          ORDER BY count DESC
          LIMIT 10`,
    args: []
  });
  const topCampaigns = topCampaignsResult.rows.map((row) => ({
    source: (row.utm_source as string | null) ?? null,
    medium: (row.utm_medium as string | null) ?? null,
    campaign: (row.utm_campaign as string | null) ?? null,
    count: row.count as number
  }));

  // Downloads: every door records one row under `/api/downloads/<asset>` via
  // `trackDownload`, so the asset is the path suffix and nothing has to be
  // parsed out of a query string.
  const downloadsResult = await conn.execute({
    sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics
          WHERE path LIKE '${DOWNLOAD_PATH_PREFIX}%'
          AND created_at >= datetime('now', '-${days} days')
          GROUP BY path
          ORDER BY count DESC`,
    args: []
  });
  const downloads = summariseDownloads(
    downloadsResult.rows.map((row) => ({
      path: String(row.path),
      count: row.count as number
    }))
  );

  return {
    totalVisits,
    totalPageVisits,
    totalApiCalls,
    uniqueVisitors,
    uniqueUsers,
    topPages,
    topApiCalls,
    topReferrers,
    deviceTypes,
    browsers,
    topCampaigns,
    downloads
  };
}

export async function getPathAnalytics(
  path: string,
  days: number = 30
): Promise<{
  totalVisits: number;
  uniqueVisitors: number;
  avgDurationMs: number | null;
  visitsByDay: Array<{ date: string; count: number }>;
}> {
  const conn = ConnectionFactory();

  const totalVisitsResult = await conn.execute({
    sql: `SELECT COUNT(*) as count FROM VisitorAnalytics 
          WHERE path = ? 
          AND created_at >= datetime('now', '-${days} days')`,
    args: [path]
  });
  const totalVisits = (totalVisitsResult.rows[0]?.count as number) || 0;

  const uniqueVisitorsResult = await conn.execute({
    sql: `SELECT COUNT(DISTINCT ip_address) as count FROM VisitorAnalytics 
          WHERE path = ? 
          AND ip_address IS NOT NULL 
          AND created_at >= datetime('now', '-${days} days')`,
    args: [path]
  });
  const uniqueVisitors = (uniqueVisitorsResult.rows[0]?.count as number) || 0;

  const avgDurationResult = await conn.execute({
    sql: `SELECT AVG(duration_ms) as avg FROM VisitorAnalytics 
          WHERE path = ? 
          AND duration_ms IS NOT NULL 
          AND created_at >= datetime('now', '-${days} days')`,
    args: [path]
  });
  const avgDurationMs = avgDurationResult.rows[0]?.avg as number | null;

  const visitsByDayResult = await conn.execute({
    sql: `SELECT DATE(created_at) as date, COUNT(*) as count 
          FROM VisitorAnalytics 
          WHERE path = ? 
          AND created_at >= datetime('now', '-${days} days')
          GROUP BY DATE(created_at) 
          ORDER BY date DESC`,
    args: [path]
  });
  const visitsByDay = visitsByDayResult.rows.map((row) => ({
    date: row.date as string,
    count: row.count as number
  }));

  return {
    totalVisits,
    uniqueVisitors,
    avgDurationMs,
    visitsByDay
  };
}

export async function getPerformanceStats(days: number = 30): Promise<{
  avgLcp: number | null;
  avgFcp: number | null;
  avgCls: number | null;
  avgInp: number | null;
  avgTtfb: number | null;
  avgDomLoad: number | null;
  avgLoadComplete: number | null;
  p75Lcp: number | null;
  p75Fcp: number | null;
  totalWithMetrics: number;
  byPath: Array<{
    path: string;
    avgLcp: number;
    avgFcp: number;
    avgCls: number;
    avgTtfb: number;
    count: number;
  }>;
}> {
  const conn = ConnectionFactory();

  const avgResult = await conn.execute({
    sql: `SELECT 
      AVG(lcp) as avgLcp,
      AVG(fcp) as avgFcp,
      AVG(cls) as avgCls,
      AVG(inp) as avgInp,
      AVG(ttfb) as avgTtfb,
      AVG(dom_load) as avgDomLoad,
      AVG(load_complete) as avgLoadComplete,
      COUNT(*) as total
    FROM VisitorAnalytics 
    WHERE created_at >= datetime('now', '-${days} days')
    AND fcp IS NOT NULL`,
    args: []
  });

  const avgRow = avgResult.rows[0] as any;

  // Get 75th percentile for LCP and FCP (approximation using median)
  const p75LcpResult = await conn.execute({
    sql: `SELECT lcp as p75
    FROM VisitorAnalytics 
    WHERE created_at >= datetime('now', '-${days} days')
    AND lcp IS NOT NULL
    ORDER BY lcp
    LIMIT 1 OFFSET (
      SELECT COUNT(*) * 75 / 100 
      FROM VisitorAnalytics 
      WHERE created_at >= datetime('now', '-${days} days')
      AND lcp IS NOT NULL
    )`,
    args: []
  });

  const p75FcpResult = await conn.execute({
    sql: `SELECT fcp as p75
    FROM VisitorAnalytics 
    WHERE created_at >= datetime('now', '-${days} days')
    AND fcp IS NOT NULL
    ORDER BY fcp
    LIMIT 1 OFFSET (
      SELECT COUNT(*) * 75 / 100 
      FROM VisitorAnalytics 
      WHERE created_at >= datetime('now', '-${days} days')
      AND fcp IS NOT NULL
    )`,
    args: []
  });

  const byPathResult = await conn.execute({
    sql: `SELECT 
      path,
      AVG(lcp) as avgLcp,
      AVG(fcp) as avgFcp,
      AVG(cls) as avgCls,
      AVG(ttfb) as avgTtfb,
      COUNT(*) as count
    FROM VisitorAnalytics 
    WHERE created_at >= datetime('now', '-${days} days')
    AND fcp IS NOT NULL
    AND path NOT LIKE '/api/%'
    GROUP BY path
    ORDER BY count DESC
    LIMIT 20`,
    args: []
  });

  const byPath = byPathResult.rows.map((row: any) => ({
    path: row.path,
    avgLcp: row.avgLcp || 0,
    avgFcp: row.avgFcp || 0,
    avgCls: row.avgCls || 0,
    avgTtfb: row.avgTtfb || 0,
    count: row.count
  }));

  return {
    avgLcp: avgRow?.avgLcp || null,
    avgFcp: avgRow?.avgFcp || null,
    avgCls: avgRow?.avgCls || null,
    avgInp: avgRow?.avgInp || null,
    avgTtfb: avgRow?.avgTtfb || null,
    avgDomLoad: avgRow?.avgDomLoad || null,
    avgLoadComplete: avgRow?.avgLoadComplete || null,
    p75Lcp: (p75LcpResult.rows[0] as any)?.p75 || null,
    p75Fcp: (p75FcpResult.rows[0] as any)?.p75 || null,
    totalWithMetrics: avgRow?.total || 0,
    byPath
  };
}

/**
 * Delete visitor analytics older than `olderThanDays`. Reachable from tRPC
 * (admin) and from the daily retention cron (`/api/analytics/retention`);
 * the SQL and the input rules live in `./analytics-retention`.
 */
export async function cleanupOldAnalytics(
  olderThanDays: number
): Promise<number> {
  return pruneVisitorAnalytics(ConnectionFactory(), olderThanDays);
}

function parseUserAgent(userAgent?: string): {
  deviceType: string | null;
  browser: string | null;
  os: string | null;
} {
  if (!userAgent) {
    return { deviceType: null, browser: null, os: null };
  }

  const ua = userAgent.toLowerCase();

  let deviceType: string | null = "desktop";
  if (ua.includes("mobile")) deviceType = "mobile";
  else if (ua.includes("tablet") || ua.includes("ipad")) deviceType = "tablet";

  let browser: string | null = null;
  if (ua.includes("edg")) browser = "edge";
  else if (ua.includes("chrome")) browser = "chrome";
  else if (ua.includes("firefox")) browser = "firefox";
  else if (ua.includes("safari") && !ua.includes("chrome")) browser = "safari";
  else if (ua.includes("opera") || ua.includes("opr")) browser = "opera";

  let os: string | null = null;
  if (ua.includes("windows")) os = "windows";
  else if (ua.includes("mac")) os = "macos";
  else if (ua.includes("linux")) os = "linux";
  else if (ua.includes("android")) os = "android";
  else if (ua.includes("iphone") || ua.includes("ipad")) os = "ios";

  return { deviceType, browser, os };
}

/**
 * Fill in what the writer did not supply, and apply the privacy floor.
 *
 * Every analytics entry — page views, tRPC context, downloads, checkout —
 * passes through here, which is why the truncation lives here and not at the
 * call sites: a future writer cannot forget it, and no caller can persist a
 * full client address by accident (FRE-369, finding Q4). The audit log
 * (`./audit.ts`) is a separate, pre-existing surface with its own stakes and is
 * deliberately untouched by this change.
 */
export function enrichAnalyticsEntry(entry: AnalyticsEntry): AnalyticsEntry {
  const { deviceType, browser, os } = parseUserAgent(
    entry.userAgent || undefined
  );

  return {
    ...entry,
    ipAddress: truncateIpAddress(entry.ipAddress),
    deviceType: entry.deviceType || deviceType,
    browser: entry.browser || browser,
    os: entry.os || os
  };
}