downloads: track every door and surface them on /analytics

The analytics page could not answer "how many downloads, of which
product". Three of the four download doors recorded nothing: the tRPC
click proxy only left the generic request row, whose path is the raw URL
with the asset buried in `?batch=1&input={"asset_name":...}`, and the S3
file route that serves Sparkle updates and the Nook app logged nothing
at all. The one door that did log (the Nook redirect) kept a private copy
of the logVisit/enrich/flush sequence, under its own path.

All doors now go through trackDownload, which writes one row per event
under /api/downloads/<asset> and flushes it inside the request — a click
is low-volume and serverless is not guaranteed to run the 30s batch
timer after the response. The file route maps its prefix allowlist to
asset keys, so a version bump stays one asset rather than one per
release. getAnalyticsSummary groups those rows into a downloads block and
the freno.me panel renders it under the overview cards.

History is recovered to the degree it exists: the Nook door's rows and
the tRPC clicks still carry enough to derive from (asset name in the URL).
scripts/backfill-downloads.ts inserts those derived rows with a
`backfill-<source id>` primary key and INSERT OR IGNORE, leaving the
originals untouched, so re-running is a no-op. Run against the live table
it added 66 rows: gaze 42, thenook 19, inputhalo 4, lineage 1.

Tests: new backfill suite (derivation, visitor detail carried over,
idempotency, empty history) and summariseDownloads cases; the Nook route
test now pins the canonical path, and the NOOK-337 allowlist audit reads
the map keys instead of quoted strings. Red-checked by stubbing the
downloads block back to zeros — the summary case fails, then passes.
Full suite matches master.

Co-Authored-By: nw-flash <noreply@nw-flash.local>
MISC-48

src/server/analytics.ts

@@ -1,6 +1,7 @@
 import { ConnectionFactory } from "./database";
 import { v4 as uuid } from "uuid";
 import type { VisitorAnalytics, AnalyticsQuery } from "~/db/types";
+import { DOWNLOAD_PATH_PREFIX, summariseDownloads } from "~/lib/download-asset";
 import { CACHE_CONFIG } from "~/config";
 import {
   VISITOR_ANALYTICS_ADDED_COLUMNS,
@@ -272,6 +273,10 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
   topReferrers: Array<{ referrer: string; count: number }>;
   deviceTypes: Array<{ type: string; count: number }>;
   browsers: Array<{ browser: string; count: number }>;
+  downloads: {
+    total: number;
+    byAsset: Array<{ asset: string; count: number }>;
+  };
   topCampaigns: Array<{
     source: string | null;
     medium: string | null;
@@ -409,6 +414,24 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
     count: row.count as number
   }));
 
+  // Downloads: every door records one row under `/api/downloads/<asset>` via
+  // `trackDownload`, so the asset is the path suffix and nothing has to be
+  // parsed out of a query string.
+  const downloadsResult = await conn.execute({
+    sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics
+          WHERE path LIKE '${DOWNLOAD_PATH_PREFIX}%'
+          AND created_at >= datetime('now', '-${days} days')
+          GROUP BY path
+          ORDER BY count DESC`,
+    args: []
+  });
+  const downloads = summariseDownloads(
+    downloadsResult.rows.map((row) => ({
+      path: String(row.path),
+      count: row.count as number
+    }))
+  );
+
   return {
     totalVisits,
     totalPageVisits,
@@ -420,7 +443,8 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
     topReferrers,
     deviceTypes,
     browsers,
-    topCampaigns
+    topCampaigns,
+    downloads
   };
 }
 

src/server/api/routers/downloads.ts

@@ -8,6 +8,7 @@ import {
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { env } from "~/env/server";
 import { TRPCError } from "@trpc/server";
+import { trackDownload } from "~/server/track-download";
 
 const assets: Record<string, string> = {
   lineage: "Life and Lineage.apk",
@@ -73,7 +74,14 @@ const latestAssets: Record<
 export const downloadsRouter = createTRPCRouter({
   getDownloadUrl: publicProcedure
     .input(z.object({ asset_name: z.string() }))
-    .query(async ({ input }) => {
+    .query(async ({ input, ctx }) => {
+      // Logged before signing: a click on a door that then fails to sign is
+      // exactly the event the funnel needs to show. Unknown asset names are
+      // skipped rather than recorded as a download that never happened.
+      if (assets[input.asset_name] || latestAssets[input.asset_name]) {
+        await trackDownload(ctx.event, input.asset_name);
+      }
+
       const bucket = env.VITE_DOWNLOAD_BUCKET_STRING;
 
       const credentials = {

src/server/backfill-downloads.test.ts

@@ -0,0 +1,186 @@
+import { beforeEach, describe, expect, it } from "bun:test";
+import { Database } from "bun:sqlite";
+import {
+  VISITOR_ANALYTICS_COLUMNS,
+  VISITOR_ANALYTICS_DDL,
+  VISITOR_ANALYTICS_INDEXES,
+  VISITOR_ANALYTICS_INSERT
+} from "./analytics-schema";
+import { summariseDownloads } from "~/lib/download-asset";
+import { assetFromLegacyPath, backfillDownloads } from "./backfill-downloads";
+
+// Why a real in-memory SQLite database rather than a stub: the backfill runs
+// real SQL against the live table shape, and a stub would happily accept a
+// column list the real table does not have. Same reasoning as
+// analytics-schema.test.ts.
+//
+// Nothing is module-mocked here on purpose: bun's `mock.module` is process-wide,
+// so stubbing the connection factory would leak into the rate-limit and CSRF
+// suites that share the same database module.
+
+let db: Database;
+
+/** Adapt bun:sqlite to the libsql `execute(stmt | sql)` contract. */
+const conn = {
+  async execute(query: string | { sql: string; args?: unknown[] }) {
+    const sql = typeof query === "string" ? query : query.sql;
+    const args = (
+      typeof query === "string" ? [] : (query.args ?? [])
+    ) as never[];
+    const statement = db.query(sql);
+    if (/^\s*(select|pragma|with)\b/i.test(sql)) {
+      return { rows: statement.all(...args) as unknown[], rowsAffected: 0 };
+    }
+    return {
+      rows: [] as unknown[],
+      rowsAffected: statement.run(...args).changes
+    };
+  }
+};
+
+const LEGACY_NOOK = "/api/the-nook/download";
+
+const trpcClick = (asset: string) =>
+  `/api/trpc/downloads.getDownloadUrl?batch=1&input=%7B%220%22%3A%7B%22asset_name%22%3A%22${asset}%22%7D%7D`;
+
+/** Columns whose value varies per row; everything else stays NULL. */
+const fixedColumns: Record<string, string> = {
+  method: "GET",
+  referrer: "https://gaze.freno.me/",
+  user_agent: "Mozilla/5.0 (Macintosh)",
+  ip_address: "203.0.113.7",
+  country: "GB"
+};
+
+function insertLegacy(id: string, path: string) {
+  const values = VISITOR_ANALYTICS_COLUMNS.map((column) => {
+    if (column === "id") return id;
+    if (column === "path") return path;
+    return fixedColumns[column] ?? null;
+  });
+  db.query(VISITOR_ANALYTICS_INSERT).run(...(values as never[]));
+}
+
+function canonicalPaths(): string[] {
+  return db
+    .query<{ path: string }, []>(
+      `SELECT path FROM VisitorAnalytics WHERE path LIKE '/api/downloads/%' ORDER BY path`
+    )
+    .all()
+    .map((r: { path: string }) => r.path);
+}
+
+function legacyRowCount(): number {
+  return db
+    .query<
+      { n: number },
+      [string]
+    >(`SELECT COUNT(*) AS n FROM VisitorAnalytics WHERE path = ?`)
+    .get(LEGACY_NOOK)!.n;
+}
+
+beforeEach(() => {
+  db = new Database(":memory:");
+  db.exec(VISITOR_ANALYTICS_DDL);
+  for (const index of VISITOR_ANALYTICS_INDEXES) db.exec(index);
+});
+
+describe("backfillDownloads", () => {
+  it("derives a canonical row from each recoverable kind of history", async () => {
+    insertLegacy("nook-1", LEGACY_NOOK);
+    insertLegacy("gaze-1", trpcClick("gaze"));
+    insertLegacy("halo-1", trpcClick("inputhalo"));
+    insertLegacy("page-1", "/nook/");
+
+    const added = Object.fromEntries(
+      (await backfillDownloads(conn)).map((r) => [r.asset, r.added])
+    );
+
+    expect(added).toEqual({ thenook: 1, gaze: 1, inputhalo: 1 });
+    expect(canonicalPaths()).toEqual([
+      "/api/downloads/gaze",
+      "/api/downloads/inputhalo",
+      "/api/downloads/thenook"
+    ]);
+    // A page visit is not download history and gains no derived row.
+    expect(
+      db
+        .query(`SELECT * FROM VisitorAnalytics WHERE id = 'backfill-page-1'`)
+        .all()
+    ).toHaveLength(0);
+  });
+
+  it("keeps the visitor detail of the row it derives from", async () => {
+    insertLegacy("nook-1", LEGACY_NOOK);
+
+    await backfillDownloads(conn);
+
+    expect(
+      db
+        .query<
+          { country: string; referrer: string },
+          []
+        >(`SELECT country, referrer FROM VisitorAnalytics WHERE id = 'backfill-nook-1'`)
+        .get()
+    ).toEqual({ country: "GB", referrer: "https://gaze.freno.me/" });
+  });
+
+  it("is idempotent and never rewrites the source rows", async () => {
+    insertLegacy("nook-1", LEGACY_NOOK);
+
+    await backfillDownloads(conn);
+    const second = await backfillDownloads(conn);
+
+    expect(second).toEqual([]);
+    expect(canonicalPaths()).toHaveLength(1);
+    expect(legacyRowCount()).toBe(1);
+  });
+
+  it("derives nothing when there is no legacy history", async () => {
+    expect(await backfillDownloads(conn)).toEqual([]);
+    expect(canonicalPaths()).toEqual([]);
+  });
+});
+
+describe("assetFromLegacyPath", () => {
+  it("maps the Nook door to its canonical asset", () => {
+    expect(assetFromLegacyPath(LEGACY_NOOK)).toBe("thenook");
+  });
+
+  it("reads the asset out of an encoded or a plain tRPC request URL", () => {
+    expect(assetFromLegacyPath(trpcClick("gaze"))).toBe("gaze");
+    expect(
+      assetFromLegacyPath(
+        '/api/trpc/downloads.getDownloadUrl?batch=1&input={"0":{"asset_name":"lineage"}}'
+      )
+    ).toBe("lineage");
+  });
+
+  it("returns null when there is nothing to derive", () => {
+    expect(assetFromLegacyPath("/nook/")).toBeNull();
+    expect(
+      assetFromLegacyPath("/api/trpc/downloads.getDownloadUrl")
+    ).toBeNull();
+  });
+});
+
+describe("summariseDownloads", () => {
+  it("labels each row by the asset in its path and totals them", () => {
+    expect(
+      summariseDownloads([
+        { path: "/api/downloads/gaze", count: 4 },
+        { path: "/api/downloads/thenook", count: 1 }
+      ])
+    ).toEqual({
+      total: 5,
+      byAsset: [
+        { asset: "gaze", count: 4 },
+        { asset: "thenook", count: 1 }
+      ]
+    });
+  });
+
+  it("reports nothing rather than throwing on an empty table", () => {
+    expect(summariseDownloads([])).toEqual({ total: 0, byAsset: [] });
+  });
+});

src/server/backfill-downloads.ts

@@ -0,0 +1,95 @@
+import { VISITOR_ANALYTICS_COLUMNS } from "~/server/analytics-schema";
+import { downloadPath } from "~/lib/download-asset";
+
+/**
+ * Backfill canonical `downloadPath(asset)` rows from download history that was
+ * recorded before `trackDownload` existed.
+ *
+ * Two shapes of history are recoverable, and only these two:
+ *
+ *  - the Nook door's own rows (`/api/the-nook/download`), and
+ *  - tRPC clicks, where the asset name still sits in the request URL as
+ *    `?input={"0":{"asset_name":"gaze"}}` (percent-encoded in practice).
+ *
+ * Every other door wrote nothing, so there is nothing to derive for them.
+ *
+ * Derived rows keep the source row's timestamp, visitor, campaign and country,
+ * get `backfill-<source id>` as primary key and are written with
+ * `INSERT OR IGNORE` — so a second run adds nothing, and the original rows are
+ * left untouched (they are genuine API traffic, and the analytics page only
+ * counts the canonical path).
+ */
+
+const LEGACY_NOOK_PATH = "/api/the-nook/download";
+const TRPC_DOOR_PREFIX = "/api/trpc/downloads.getDownloadUrl";
+
+/** The two libsql calls used here, narrow enough for an in-memory test client. */
+export interface DownloadBackfillConn {
+  execute(query: {
+    sql: string;
+    // `any[]` not `unknown[]`: the libsql client types its bind args as InArg[],
+    // and a narrower array type here would reject the real connection.
+    args?: any[];
+  }): Promise<{ rows: unknown[]; rowsAffected?: number }>;
+}
+
+export interface DownloadBackfillResult {
+  asset: string;
+  added: number;
+}
+
+/** One statement: copy every column of the source row, override id and path. */
+const INSERT_DERIVED = `INSERT OR IGNORE INTO VisitorAnalytics (${VISITOR_ANALYTICS_COLUMNS.join(", ")})
+SELECT ${VISITOR_ANALYTICS_COLUMNS.map((column) =>
+  column === "id"
+    ? "'backfill-' || id"
+    : column === "path"
+      ? "?"
+      : `"${column}"`
+).join(", ")}
+FROM VisitorAnalytics
+WHERE id = ?`;
+
+/** Pull the asset name back out of a logged tRPC request URL. */
+export function assetFromLegacyPath(path: string): string | null {
+  if (path === LEGACY_NOOK_PATH) return "thenook";
+
+  let decoded = path;
+  try {
+    decoded = decodeURIComponent(path);
+  } catch {
+    // A malformed escape is not a reason to drop the row — match on the raw.
+  }
+
+  return /"asset_name"\s*:\s*"([^"]+)"/.exec(decoded)?.[1] ?? null;
+}
+
+export async function backfillDownloads(
+  conn: DownloadBackfillConn
+): Promise<DownloadBackfillResult[]> {
+  const legacy = await conn.execute({
+    sql: `SELECT id, path FROM VisitorAnalytics WHERE path = ? OR path LIKE ?`,
+    args: [LEGACY_NOOK_PATH, `${TRPC_DOOR_PREFIX}%`]
+  });
+
+  const added = new Map<string, number>();
+
+  for (const row of legacy.rows) {
+    const { id, path } = row as { id: string; path: string };
+    const asset = assetFromLegacyPath(path);
+    if (!asset) continue;
+
+    const result = await conn.execute({
+      sql: INSERT_DERIVED,
+      args: [downloadPath(asset), id]
+    });
+
+    if (result.rowsAffected) {
+      added.set(asset, (added.get(asset) ?? 0) + result.rowsAffected);
+    }
+  }
+
+  return [...added.entries()]
+    .map(([asset, count]) => ({ asset, added: count }))
+    .sort((a, b) => b.added - a.added);
+}

src/server/track-download.ts

@@ -0,0 +1,61 @@
+import type { APIEvent } from "@solidjs/start/server";
+import { getRequestIP } from "vinxi/http";
+import {
+  logVisit,
+  enrichAnalyticsEntry,
+  flushAnalytics
+} from "~/server/analytics";
+import { parseUtm } from "~/server/utm";
+import { downloadPath } from "~/lib/download-asset";
+
+/**
+ * Record one download event under the canonical path `/api/downloads/<asset>`.
+ *
+ * Every download door funnels through here so the analytics page has a single
+ * label per asset to group on: the tRPC click proxy, the S3 file route that
+ * serves Sparkle updates, and the Nook door.
+ *
+ * Best-effort by design — a failed analytics write must never block a
+ * download. Flushed rather than merely buffered: a download is low-volume and
+ * the 30s batch timer is not guaranteed to survive the response on serverless.
+ *
+ * `event.request` is expected to be a Fetch `Request` (true for Vinxi route
+ * handlers and browser tRPC calls). A server-side `createCaller` hands over a
+ * Node request, whose headers have no `get` — that throw is swallowed below,
+ * so such a click is not recorded. No server-side caller downloads today.
+ */
+export async function trackDownload(
+  event: APIEvent,
+  asset: string
+): Promise<void> {
+  const { request } = event;
+
+  try {
+    const referrer =
+      request.headers.get("referer") ?? request.headers.get("referrer");
+    const forwarded = request.headers.get("x-forwarded-for");
+
+    await logVisit(
+      enrichAnalyticsEntry({
+        path: downloadPath(asset),
+        method: "GET",
+        referrer,
+        userAgent: request.headers.get("user-agent"),
+        ipAddress:
+          getRequestIP(event.nativeEvent) ??
+          (forwarded ? forwarded.split(",")[0]!.trim() : null),
+        // The click carries its own tags when a campaign links straight at the
+        // door; otherwise the landing page's tags ride along in the referer.
+        ...parseUtm(request.url, referrer),
+        // Edge header country, overridable with ?c= for manual verification.
+        country:
+          new URL(request.url).searchParams.get("c") ??
+          request.headers.get("x-vercel-ip-country") ??
+          request.headers.get("cf-ipcountry")
+      })
+    );
+    await flushAnalytics();
+  } catch (error) {
+    console.error(`Failed to log download of ${asset}:`, error);
+  }
+}