diff --git a/src/db/types.ts b/src/db/types.ts index 759ac8d..acd1a4f 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -148,6 +148,9 @@ export interface VisitorAnalytics { device_type?: string | null; browser?: string | null; os?: string | null; + utm_source?: string | null; + utm_medium?: string | null; + utm_campaign?: string | null; duration_ms?: number | null; fcp?: number | null; lcp?: number | null; diff --git a/src/routes/api/the-nook/_download.ts b/src/routes/api/the-nook/_download.ts new file mode 100644 index 0000000..6903fe5 --- /dev/null +++ b/src/routes/api/the-nook/_download.ts @@ -0,0 +1,75 @@ +import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3"; +import { env } from "~/env/server"; + +/** + * Download resolution for The Nook, kept out of the route module on purpose. + * + * Vinxi's route tree-shake (`vinxi/lib/plugins/tree-shake.babel.js`) deletes + * every exported declaration in a route file whose name is not in the + * `?pick=` list the router asks for — and a route is picked as `pick=GET`. + * A helper exported from `download.ts` is therefore removed while `GET` + * still references it, which fails at runtime with "is not defined" in dev + * and in the production bundle alike. Only HTTP handlers may be exported + * from a route; helpers live here, where the transform never runs. Same + * reason `_lib.ts` exists. + */ + +const APPCast_KEY = "api/TheNook/appcast.xml"; + +/** Kept in step with the appcast route's own 5-minute cache. */ +const RESOLVE_TTL_MS = 5 * 60 * 1000; + +const ENCLOSURE_RE = /]*\burl="([^"]+)"/gi; + +/** + * Picks the newest release artifact out of a Sparkle appcast: the first + * enclosure that is a .dmg, else the first .zip, else the first enclosure + * of any kind. Enclosures appear in newest-first order. + */ +export function pickEnclosure(xml: string): string | null { + const urls: string[] = []; + for (const match of xml.matchAll(ENCLOSURE_RE)) { + if (match[1]) urls.push(match[1]); + } + return ( + urls.find((u) => /\.dmg(\?|$)/i.test(u)) ?? + urls.find((u) => /\.zip(\?|$)/i.test(u)) ?? + urls[0] ?? + null + ); +} + +let cached: { url: string; at: number } | null = null; + +/** Drops the cached enclosure so the next request re-reads the appcast. */ +export function invalidateDownloadCache(): void { + cached = null; +} + +/** Resolves (and briefly caches) the current download URL from the appcast. */ +export async function resolveLatestDownload(): Promise { + if (cached && Date.now() - cached.at < RESOLVE_TTL_MS) { + return cached.url; + } + + const client = new S3Client({ + region: env.AWS_REGION, + credentials: { + accessKeyId: env.MY_AWS_ACCESS_KEY, + secretAccessKey: env.MY_AWS_SECRET_KEY + } + }); + + const response = await client.send( + new GetObjectCommand({ + Bucket: env.VITE_DOWNLOAD_BUCKET_STRING, + Key: APPCast_KEY + }) + ); + if (!response.Body) return null; + + const xml = await response.Body.transformToString(); + const url = pickEnclosure(xml); + if (url) cached = { url, at: Date.now() }; + return url; +} diff --git a/src/routes/api/the-nook/download.test.ts b/src/routes/api/the-nook/download.test.ts new file mode 100644 index 0000000..5186628 --- /dev/null +++ b/src/routes/api/the-nook/download.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it, mock, beforeEach } from "bun:test"; + +// The route's contracts: +// 1. Every click is recorded through the shared analytics buffer with the +// path, referrer, country and device type the funnel needs. +// 2. The response is a no-store 302 to the newest release enclosure, so a +// CDN can never serve a later click without it reaching this handler. +// 3. An unreadable appcast degrades to 503 — but the click is still logged, +// because a failed download is exactly the event worth seeing. + +type Entry = Record; +const logged: Entry[] = []; +let appcastXml = ""; +let s3ShouldThrow = false; +let flushCalls = 0; + +mock.module("~/env/server", () => ({ + env: { + AWS_REGION: "us-east-1", + MY_AWS_ACCESS_KEY: "test-key", + MY_AWS_SECRET_KEY: "test-secret", + VITE_DOWNLOAD_BUCKET_STRING: "test-bucket" + } +})); + +mock.module("@aws-sdk/client-s3", () => ({ + S3Client: class { + async send() { + if (s3ShouldThrow) throw new Error("s3 unavailable"); + return { + Body: appcastXml + ? { transformToString: async () => appcastXml } + : null + }; + } + }, + GetObjectCommand: class { + input: unknown; + constructor(input: unknown) { + this.input = input; + } + } +})); + +mock.module("vinxi/http", () => ({ + getRequestIP: () => null +})); + +mock.module("~/server/analytics", () => ({ + logVisit: async (entry: Entry) => { + logged.push(entry); + }, + // The route must flush, not merely buffer: the batch timer is not + // guaranteed to survive the response on serverless. + flushAnalytics: async () => { + flushCalls += 1; + }, + // Mirrors the real enrichment contract: deviceType is derived from the UA. + enrichAnalyticsEntry: (entry: Entry) => ({ + ...entry, + deviceType: String(entry.userAgent ?? "").includes("Macintosh") + ? "desktop" + : null + }) +})); + +const { GET } = await import("./download"); +const { pickEnclosure, invalidateDownloadCache } = await import("./_download"); + +const DMG = "https://cdn.example/TheNook-0.6.3.dmg"; +const ZIP = "https://cdn.example/TheNook-0.6.3.zip"; + +const appcast = (urls: string[]) => + `` + + urls + .map( + (u) => + `` + ) + .join("") + + ``; + +function request(headers: Record = {}, query = "") { + return { + request: new Request( + `https://nook.freno.me/api/the-nook/download${query}`, + { headers } + ), + nativeEvent: {} + } as never; +} + +const MAC_UA = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15"; + +beforeEach(() => { + logged.length = 0; + appcastXml = appcast([DMG, ZIP]); + s3ShouldThrow = false; + flushCalls = 0; + invalidateDownloadCache(); +}); + +describe("GET /api/the-nook/download", () => { + it("logs the click and 302s to the newest dmg", async () => { + const res = await GET(request({ referer: "https://nook.freno.me/" })); + + expect(res.status).toBe(302); + expect(res.headers.get("Location")).toBe(DMG); + expect(res.headers.get("Cache-Control")).toBe("no-store"); + + expect(logged.length).toBe(1); + expect(logged[0]).toMatchObject({ + path: "/api/the-nook/download", + method: "GET", + referrer: "https://nook.freno.me/" + }); + // Flushed within the request, not left to the batch timer. + expect(flushCalls).toBe(1); + }); + + it("derives the device type from the user agent", async () => { + await GET(request({ "user-agent": MAC_UA })); + expect(logged[0]!.deviceType).toBe("desktop"); + }); + + it("records the edge country, and lets ?c= override it", async () => { + await GET(request({ "x-vercel-ip-country": "GB" })); + expect(logged[0]!.country).toBe("GB"); + + invalidateDownloadCache(); + await GET(request({ "cf-ipcountry": "DE" })); + expect(logged[1]!.country).toBe("DE"); + + invalidateDownloadCache(); + await GET(request({ "x-vercel-ip-country": "GB" }, "?c=JP")); + expect(logged[2]!.country).toBe("JP"); + }); + + it("falls back to the forwarded address when no request IP is available", async () => { + await GET(request({ "x-forwarded-for": "198.51.100.9, 10.0.0.1" })); + expect(logged[0]!.ipAddress).toBe("198.51.100.9"); + }); + + it("still logs the click when the appcast cannot be read", async () => { + s3ShouldThrow = true; + const res = await GET(request()); + + expect(res.status).toBe(503); + expect(res.headers.get("Cache-Control")).toBe("no-store"); + expect(logged.length).toBe(1); + }); + + it("503s when the appcast carries no enclosure", async () => { + appcastXml = ""; + invalidateDownloadCache(); + + const res = await GET(request()); + expect(res.status).toBe(503); + }); + + it("captures campaign tags from the click URL and the referer", async () => { + await GET( + request( + { referer: "https://nook.freno.me/?utm_source=hn&utm_campaign=launch" }, + "?utm_medium=post" + ) + ); + + expect(logged[0]).toMatchObject({ + utmSource: "hn", + utmMedium: "post", + utmCampaign: "launch" + }); + }); + + it("logs nulls when nothing is tagged", async () => { + await GET(request({ referer: "https://nook.freno.me/" })); + expect(logged[0]).toMatchObject({ + utmSource: null, + utmMedium: null, + utmCampaign: null + }); + }); +}); + +describe("pickEnclosure", () => { + it("prefers a dmg over a zip even when the zip comes first", () => { + expect(pickEnclosure(appcast([ZIP, DMG]))).toBe(DMG); + }); + + it("falls back to a zip, then to any enclosure", () => { + expect(pickEnclosure(appcast([ZIP]))).toBe(ZIP); + expect(pickEnclosure(appcast(["https://cdn.example/notes.txt"]))).toBe( + "https://cdn.example/notes.txt" + ); + }); + + it("returns null when there is no enclosure", () => { + expect(pickEnclosure("")).toBeNull(); + }); +}); diff --git a/src/routes/api/the-nook/download.ts b/src/routes/api/the-nook/download.ts new file mode 100644 index 0000000..0613909 --- /dev/null +++ b/src/routes/api/the-nook/download.ts @@ -0,0 +1,94 @@ +import type { APIEvent } from "@solidjs/start/server"; +import { getRequestIP } from "vinxi/http"; +import { logVisit, enrichAnalyticsEntry, flushAnalytics } from "~/server/analytics"; +import { parseUtm } from "~/server/utm"; +import { resolveLatestDownload } from "./_download"; + +/** + * GET /api/the-nook/download + * + * The measured download door. Both hero buttons point here instead of + * resolving the appcast in the browser: this route resolves the latest + * release enclosure server-side, records the click through the shared + * analytics buffer, then 302s to the DMG. + * + * Why a redirect and not a direct S3 link: the appcast route and the DMG + * route never logged anything, so step 2 of the funnel (download click → + * DMG served) was invisible. Every click now lands one row in + * VisitorAnalytics under the path below, which is what `topApiCalls` + * already surfaces. + * + * Country is read from the edge headers Vercel/Cloudflare set; the + * analytics buffer carries a `country` column that nothing populated + * until now. `?c=` is accepted as an override for previews and manual + * verification. + * + * ONLY the handler is exported here: Vinxi's route tree-shake deletes + * exported declarations that are not in the router's `pick=` list, so a + * helper exported from this file would be removed while `GET` still + * referenced it. All resolution helpers live in `./_download.ts`. + */ + +function firstHeader(request: Request, ...names: string[]): string | null { + for (const name of names) { + const value = request.headers.get(name); + if (value) return value; + } + return null; +} + +export async function GET(event: APIEvent): Promise { + const { request } = event; + const url = new URL(request.url); + + const forwarded = firstHeader(request, "x-forwarded-for"); + const ipAddress = + getRequestIP(event.nativeEvent) ?? + (forwarded ? forwarded.split(",")[0]!.trim() : null); + + // Best-effort: a failed analytics write must never block a download. + // Flushed (not merely buffered) because a download click is low-volume and + // the batch timer is not guaranteed to survive the response on serverless. + try { + const referrer = firstHeader(request, "referer", "referrer"); + await logVisit( + enrichAnalyticsEntry({ + path: "/api/the-nook/download", + method: "GET", + referrer, + userAgent: firstHeader(request, "user-agent"), + ipAddress, + // The click carries its own tags when a campaign links straight here; + // otherwise the landing page's tags ride along in the referer. + ...parseUtm(request.url, referrer), + country: + url.searchParams.get("c") ?? + firstHeader(request, "x-vercel-ip-country", "cf-ipcountry") + }) + ); + await flushAnalytics(); + } catch (error) { + console.error("Failed to log download click:", error); + } + + let target: string | null = null; + try { + target = await resolveLatestDownload(); + } catch (error) { + console.error("Failed to resolve Nook download:", error); + } + + if (!target) { + return new Response("Download unavailable", { + status: 503, + headers: { "Content-Type": "text/plain", "Cache-Control": "no-store" } + }); + } + + // No-store is load-bearing: a cached 302 would serve later clicks without + // ever reaching this handler, silently dropping them from the funnel. + return new Response(null, { + status: 302, + headers: { Location: target, "Cache-Control": "no-store" } + }); +} diff --git a/src/routes/nook/index.tsx b/src/routes/nook/index.tsx index 66b7048..d774865 100644 --- a/src/routes/nook/index.tsx +++ b/src/routes/nook/index.tsx @@ -9,19 +9,14 @@ import { createSignal, Show } from "solid-js"; import { useDarkMode } from "~/context/darkMode"; import { useSite } from "~/context/SiteContext"; -/* The appcast's first enclosure always names the latest release dmg. */ -async function downloadNook() { - try { - const res = await fetch("/api/TheNook/appcast.xml"); - const xml = await res.text(); - const url = - xml.match(/ { + db = new Database(":memory:"); + db.run(VISITOR_ANALYTICS_DDL); + for (const index of VISITOR_ANALYTICS_INDEXES) db.run(index); +}); + +describe("VisitorAnalytics schema", () => { + it("creates the table idempotently", () => { + expect(() => db.run(VISITOR_ANALYTICS_DDL)).not.toThrow(); + expect(() => { + for (const index of VISITOR_ANALYTICS_INDEXES) db.run(index); + }).not.toThrow(); + }); + + it("defines every column the writers insert", () => { + const columns = ( + db.prepare(`PRAGMA table_info(VisitorAnalytics)`).all() as { + name: string; + }[] + ).map((c) => c.name); + + for (const column of VISITOR_ANALYTICS_COLUMNS) { + expect(columns).toContain(column); + } + // The reverse direction matters too: a column the code writes but the + // DDL omits is the failure this whole file exists to catch. + expect(VISITOR_ANALYTICS_COLUMNS.length).toBe(23); + }); + + it("has one placeholder per inserted column", () => { + const placeholders = (VISITOR_ANALYTICS_INSERT.match(/\?/g) ?? []).length; + expect(placeholders).toBe(VISITOR_ANALYTICS_COLUMNS.length); + expect(sampleArgs.length).toBe(VISITOR_ANALYTICS_COLUMNS.length); + }); + + it("accepts a full row and returns the campaign columns", () => { + db.prepare(VISITOR_ANALYTICS_INSERT).run(...sampleArgs); + + const row = db + .prepare( + `SELECT path, country, utm_source, utm_medium, utm_campaign + FROM VisitorAnalytics` + ) + .get() as Record; + + expect(row).toEqual({ + path: "/nook/", + country: "GB", + utm_source: "hn", + utm_medium: "post", + utm_campaign: "launch" + }); + }); + + it("accepts a row with no campaign tags", () => { + const args = [...sampleArgs]; + args[11] = args[12] = args[13] = null; + db.prepare(VISITOR_ANALYTICS_INSERT).run(...args); + + const row = db + .prepare(`SELECT utm_source FROM VisitorAnalytics`) + .get() as Record; + expect(row.utm_source).toBeNull(); + }); + + it("adds the post-hoc columns to a table that predates them", () => { + // Simulate the older table: create it without the utm columns, then run + // the ALTER path the bootstrap uses and confirm the writers still work. + const old = new Database(":memory:"); + const withoutUtm = VISITOR_ANALYTICS_COLUMNS.filter( + (c) => !VISITOR_ANALYTICS_ADDED_COLUMNS.some((a) => a.name === c) + ); + old.run( + `CREATE TABLE VisitorAnalytics (${withoutUtm + .map((c) => `${c} TEXT`) + .join(", ")})` + ); + + const present = new Set( + (old.prepare(`PRAGMA table_info(VisitorAnalytics)`).all() as { + name: string; + }[]).map((c) => c.name) + ); + for (const column of VISITOR_ANALYTICS_ADDED_COLUMNS) { + if (!present.has(column.name)) old.run(column.ddl); + } + + expect(() => + old.prepare(VISITOR_ANALYTICS_INSERT).run(...sampleArgs) + ).not.toThrow(); + }); +}); diff --git a/src/server/analytics-schema.ts b/src/server/analytics-schema.ts new file mode 100644 index 0000000..93d43ba --- /dev/null +++ b/src/server/analytics-schema.ts @@ -0,0 +1,123 @@ +/** + * VisitorAnalytics schema — the single source of truth for the table, its + * indexes, and the column list both writers insert. + * + * Why this file exists: `VisitorAnalytics` had no `CREATE TABLE` anywhere in + * this repo. It existed only in the live Turso database, so a fresh or + * restored environment had nothing to write into — and because + * `flushAnalyticsBuffer` swallows errors by design, every insert would fail + * silently. The definition below mirrors production exactly. + * + * `session_id` is deliberately kept. It is a leftover from the removed + * analytics-page feature (added in d2ee61b, dropped from the code in + * 58d48da) — nothing reads or writes it now, and the Nook macOS app never + * touches this table at all. SQLite would let us `DROP COLUMN`, but the + * column costs nothing and the live data in it may still be wanted, so the + * schema records it rather than removing it. + * + * Pure string constants, no imports — so the parity test can apply them to + * an in-memory SQLite database without a network or environment. + */ + +/** + * The table as it exists in production. `IF NOT EXISTS` makes applying this + * a no-op against the live database; it only does work on a database that + * has never had the table. + */ +export const VISITOR_ANALYTICS_DDL = ` + CREATE TABLE IF NOT EXISTS VisitorAnalytics ( + id TEXT PRIMARY KEY, + user_id TEXT, + path TEXT NOT NULL, + method TEXT NOT NULL, + referrer TEXT, + user_agent TEXT, + ip_address TEXT, + country TEXT, + device_type TEXT, + browser TEXT, + os TEXT, + session_id TEXT, + duration_ms INTEGER, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + fcp REAL, + lcp REAL, + cls REAL, + fid REAL, + inp REAL, + ttfb REAL, + dom_load REAL, + load_complete REAL, + utm_source TEXT, + utm_medium TEXT, + utm_campaign TEXT, + FOREIGN KEY (user_id) REFERENCES User(id) ON DELETE SET NULL + ) +`; + +/** Indexes the summary, funnel and path queries rely on. */ +export const VISITOR_ANALYTICS_INDEXES = [ + `CREATE INDEX IF NOT EXISTS idx_analytics_user_id ON VisitorAnalytics(user_id)`, + `CREATE INDEX IF NOT EXISTS idx_analytics_created_at ON VisitorAnalytics(created_at)`, + `CREATE INDEX IF NOT EXISTS idx_analytics_path ON VisitorAnalytics(path)`, + `CREATE INDEX IF NOT EXISTS idx_analytics_ip_address ON VisitorAnalytics(ip_address)`, + `CREATE INDEX IF NOT EXISTS idx_analytics_session_id ON VisitorAnalytics(session_id)`, + `CREATE INDEX IF NOT EXISTS idx_analytics_lcp ON VisitorAnalytics(lcp) WHERE lcp IS NOT NULL`, + `CREATE INDEX IF NOT EXISTS idx_analytics_fcp ON VisitorAnalytics(fcp) WHERE fcp IS NOT NULL` +]; + +/** Columns that may be missing from a table created before they existed. */ +export const VISITOR_ANALYTICS_ADDED_COLUMNS = [ + { + name: "utm_source", + ddl: `ALTER TABLE VisitorAnalytics ADD COLUMN utm_source TEXT` + }, + { + name: "utm_medium", + ddl: `ALTER TABLE VisitorAnalytics ADD COLUMN utm_medium TEXT` + }, + { + name: "utm_campaign", + ddl: `ALTER TABLE VisitorAnalytics ADD COLUMN utm_campaign TEXT` + } +]; + +/** + * Every column both writers insert, in order. The parity test asserts these + * all exist on the table, which is what stops the DDL and the INSERT from + * drifting apart again. + */ +export const VISITOR_ANALYTICS_COLUMNS = [ + "id", + "user_id", + "path", + "method", + "referrer", + "user_agent", + "ip_address", + "country", + "device_type", + "browser", + "os", + "utm_source", + "utm_medium", + "utm_campaign", + "duration_ms", + "fcp", + "lcp", + "cls", + "fid", + "inp", + "ttfb", + "dom_load", + "load_complete" +] as const; + +/** + * The one INSERT statement used by both analytics writers — the buffered + * writer in `analytics.ts` and the page-visit path in the tRPC router. + * Shared so the two can never disagree about column order or count. + */ +export const VISITOR_ANALYTICS_INSERT = `INSERT INTO VisitorAnalytics ( + ${VISITOR_ANALYTICS_COLUMNS.join(", ")} +) VALUES (${VISITOR_ANALYTICS_COLUMNS.map(() => "?").join(", ")})`; diff --git a/src/server/analytics.ts b/src/server/analytics.ts index f1d291e..eea0723 100644 --- a/src/server/analytics.ts +++ b/src/server/analytics.ts @@ -2,6 +2,43 @@ import { ConnectionFactory } from "./database"; import { v4 as uuid } from "uuid"; import type { VisitorAnalytics, AnalyticsQuery } from "~/db/types"; import { CACHE_CONFIG } from "~/config"; +import { + VISITOR_ANALYTICS_ADDED_COLUMNS, + VISITOR_ANALYTICS_DDL, + VISITOR_ANALYTICS_INDEXES, + VISITOR_ANALYTICS_INSERT +} from "./analytics-schema"; + +/** + * Idempotent schema bootstrap, mirroring `nook.ts`. The table had no DDL in + * this repo, so a database that never had it would accept no writes at all + * (and the buffer swallows its errors, so the failure would be invisible). + * Against the live database this is a no-op; it only creates what is absent. + */ +export const analyticsSchemaBootstrap: Promise = (async () => { + const conn = ConnectionFactory(); + + await conn.execute(VISITOR_ANALYTICS_DDL); + for (const index of VISITOR_ANALYTICS_INDEXES) { + await conn.execute(index); + } + + // SQLite has no "ADD COLUMN IF NOT EXISTS", so check before altering — + // covers a database restored from before these columns existed. + const info = await conn.execute(`PRAGMA table_info(VisitorAnalytics)`); + const present = new Set(info.rows.map((row) => String((row as any).name))); + for (const column of VISITOR_ANALYTICS_ADDED_COLUMNS) { + if (!present.has(column.name)) { + await conn.execute(column.ddl); + } + } +})(); + +// A bootstrap failure must not become an unhandled rejection on a request +// path that merely imports this module. +analyticsSchemaBootstrap.catch((error) => + console.error("VisitorAnalytics schema bootstrap failed:", error) +); export interface AnalyticsEntry { userId?: string | null; @@ -14,6 +51,9 @@ export interface AnalyticsEntry { deviceType?: string | null; browser?: string | null; os?: string | null; + utmSource?: string | null; + utmMedium?: string | null; + utmCampaign?: string | null; durationMs?: number | null; fcp?: number | null; lcp?: number | null; @@ -59,11 +99,7 @@ async function flushAnalyticsBuffer(): Promise { // Batch insert - more efficient than individual inserts for (const entry of entriesToWrite) { await conn.execute({ - sql: `INSERT INTO VisitorAnalytics ( - id, user_id, path, method, referrer, user_agent, ip_address, - country, device_type, browser, os, duration_ms, - fcp, lcp, cls, fid, inp, ttfb, dom_load, load_complete - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + sql: VISITOR_ANALYTICS_INSERT, args: [ uuid(), entry.userId || null, @@ -76,6 +112,9 @@ async function flushAnalyticsBuffer(): Promise { entry.deviceType || null, entry.browser || null, entry.os || null, + entry.utmSource || null, + entry.utmMedium || null, + entry.utmCampaign || null, entry.durationMs || null, entry.fcp || null, entry.lcp || null, @@ -94,6 +133,23 @@ async function flushAnalyticsBuffer(): Promise { } } +/** + * Write buffered entries now instead of waiting for the batch timer. + * + * The buffer only writes immediately at ANALYTICS_BATCH_SIZE entries; below + * that it defers to a 30-second timeout. On a long-lived server that is + * fine, but on serverless the work scheduled after a response is not + * guaranteed to run before the instance is recycled — so a low-volume but + * high-value event (a download click) could sit in memory and never be + * recorded. Callers that must not lose a row await this. + * + * Safe to call concurrently: the buffer is drained synchronously before the + * first await, so entries are never written twice. + */ +export async function flushAnalytics(): Promise { + await flushAnalyticsBuffer(); +} + /** * Schedule periodic buffer flush */ @@ -197,6 +253,9 @@ export async function queryAnalytics( device_type: row.device_type as string | null, browser: row.browser as string | null, os: row.os as string | null, + utm_source: row.utm_source as string | null, + utm_medium: row.utm_medium as string | null, + utm_campaign: row.utm_campaign as string | null, duration_ms: row.duration_ms as number | null, created_at: row.created_at as string })); @@ -213,6 +272,12 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{ topReferrers: Array<{ referrer: string; count: number }>; deviceTypes: Array<{ type: string; count: number }>; browsers: Array<{ browser: string; count: number }>; + topCampaigns: Array<{ + source: string | null; + medium: string | null; + campaign: string | null; + count: number; + }>; }> { const conn = ConnectionFactory(); @@ -324,6 +389,26 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{ count: row.count as number })); + // Campaign attribution (§8): which channel produced the traffic. Rows with + // no tags are excluded — the "untagged" bucket is a separate question and + // would otherwise dominate the list. + const topCampaignsResult = await conn.execute({ + sql: `SELECT utm_source, utm_medium, utm_campaign, COUNT(*) as count + FROM VisitorAnalytics + WHERE created_at >= datetime('now', '-${days} days') + AND (utm_source IS NOT NULL OR utm_medium IS NOT NULL OR utm_campaign IS NOT NULL) + GROUP BY utm_source, utm_medium, utm_campaign + ORDER BY count DESC + LIMIT 10`, + args: [] + }); + const topCampaigns = topCampaignsResult.rows.map((row) => ({ + source: (row.utm_source as string | null) ?? null, + medium: (row.utm_medium as string | null) ?? null, + campaign: (row.utm_campaign as string | null) ?? null, + count: row.count as number + })); + return { totalVisits, totalPageVisits, @@ -334,7 +419,8 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{ topApiCalls, topReferrers, deviceTypes, - browsers + browsers, + topCampaigns }; } diff --git a/src/server/api/routers/analytics.ts b/src/server/api/routers/analytics.ts index d7d9489..3cd6a5e 100644 --- a/src/server/api/routers/analytics.ts +++ b/src/server/api/routers/analytics.ts @@ -11,6 +11,8 @@ import { import { ConnectionFactory } from "~/server/database"; import { v4 as uuid } from "uuid"; import { getRequestIP } from "vinxi/http"; +import { parseUtm } from "~/server/utm"; +import { VISITOR_ANALYTICS_INSERT } from "~/server/analytics-schema"; /** Safely get a header value from either Fetch API Headers or Node.js IncomingHttpHeaders */ function getHeader( @@ -104,6 +106,10 @@ export const analyticsRouter = createTRPCRouter({ userAgent, referrer, ipAddress, + // The client beacon sends `pathname + search`, so the page URL's + // own query string is the campaign source; the referrer fills any + // gap (e.g. clicks that arrive on a clean path). + ...parseUtm(input.path, referrer), fcp: input.metrics.fcp, lcp: input.metrics.lcp, cls: input.metrics.cls, @@ -115,11 +121,7 @@ export const analyticsRouter = createTRPCRouter({ }); await conn.execute({ - sql: `INSERT INTO VisitorAnalytics ( - id, user_id, path, method, referrer, user_agent, ip_address, - country, device_type, browser, os, duration_ms, - fcp, lcp, cls, fid, inp, ttfb, dom_load, load_complete - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + sql: VISITOR_ANALYTICS_INSERT, args: [ uuid(), enriched.userId || null, @@ -132,6 +134,9 @@ export const analyticsRouter = createTRPCRouter({ enriched.deviceType || null, enriched.browser || null, enriched.os || null, + enriched.utmSource || null, + enriched.utmMedium || null, + enriched.utmCampaign || null, enriched.durationMs || null, enriched.fcp || null, enriched.lcp || null, diff --git a/src/server/utm.test.ts b/src/server/utm.test.ts new file mode 100644 index 0000000..b635bd6 --- /dev/null +++ b/src/server/utm.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "bun:test"; +import { parseUtm } from "./utm"; + +describe("parseUtm", () => { + it("reads the three keys from a full URL", () => { + expect( + parseUtm( + "https://nook.freno.me/?utm_source=hn&utm_medium=post&utm_campaign=launch" + ) + ).toEqual({ + utmSource: "hn", + utmMedium: "post", + utmCampaign: "launch" + }); + }); + + it("reads a relative path with a query (the client beacon's shape)", () => { + expect(parseUtm("/nook/?utm_source=r_macapps")).toEqual({ + utmSource: "r_macapps", + utmMedium: null, + utmCampaign: null + }); + }); + + it("returns all nulls when nothing is tagged", () => { + const empty = { utmSource: null, utmMedium: null, utmCampaign: null }; + expect(parseUtm("https://nook.freno.me/")).toEqual(empty); + expect(parseUtm("/nook/?ref=friend")).toEqual(empty); + expect(parseUtm(null)).toEqual(empty); + expect(parseUtm(undefined)).toEqual(empty); + expect(parseUtm("")).toEqual(empty); + }); + + it("lets an earlier argument win per key, later ones filling gaps", () => { + // A download click with no query of its own takes all three from the + // landing page carried in the referer. + expect( + parseUtm( + "https://nook.freno.me/api/the-nook/download", + "https://nook.freno.me/?utm_source=hn&utm_campaign=launch" + ) + ).toEqual({ + utmSource: "hn", + utmMedium: null, + utmCampaign: "launch" + }); + + // The click's own tags win; the referer only supplies what is missing. + expect( + parseUtm( + "https://nook.freno.me/api/the-nook/download?utm_source=x", + "https://nook.freno.me/?utm_source=hn&utm_campaign=launch" + ) + ).toEqual({ + utmSource: "x", + utmMedium: null, + utmCampaign: "launch" + }); + }); + + it("trims whitespace, drops empties, and caps length", () => { + expect(parseUtm("https://x.test/?utm_source=%20hn%20&utm_medium=")).toEqual({ + utmSource: "hn", + utmMedium: null, + utmCampaign: null + }); + + const long = parseUtm( + `https://x.test/?utm_campaign=${"a".repeat(500)}` + ); + expect(long.utmCampaign!.length).toBe(200); + }); + + it("does not case-fold values", () => { + expect(parseUtm("https://x.test/?utm_campaign=Launch").utmCampaign).toBe( + "Launch" + ); + }); + + it("ignores unrelated parameters", () => { + expect( + parseUtm("https://x.test/?utm_source=hn&c=GB&gclid=abc") + ).toEqual({ utmSource: "hn", utmMedium: null, utmCampaign: null }); + }); +}); diff --git a/src/server/utm.ts b/src/server/utm.ts new file mode 100644 index 0000000..8819ac8 --- /dev/null +++ b/src/server/utm.ts @@ -0,0 +1,82 @@ +/** + * Campaign parameters, parsed once and shared by every analytics writer. + * + * The three keys kept here are the ones a launch plan actually groups by: + * which channel (`utm_source`), through what (`utm_medium`), and which + * specific push (`utm_campaign`). Values are trimmed and length-capped, + * but never case-folded — an uppercase campaign name is a different + * campaign until the author says otherwise. + * + * `parseUtm` is variadic and first-non-empty-wins per key, which is what + * lets a logger pass both the URL it was called on and the referrer: a + * download click carries no query of its own, but the landing page it + * came from usually still does. + */ + +export interface UtmParams { + utmSource: string | null; + utmMedium: string | null; + utmCampaign: string | null; +} + +const EMPTY: UtmParams = { + utmSource: null, + utmMedium: null, + utmCampaign: null +}; + +/** Longest value we will store; far above any real campaign name. */ +const MAX_LEN = 200; + +/** + * Reads the query string out of a full URL, a path with a query, or a bare + * `a=1&b=2` fragment. Returns null when there is nothing to read. + */ +function searchOf(input: string): URLSearchParams | null { + try { + return new URL(input).searchParams; + } catch { + // Not absolute — take everything after the first "?". + const qIndex = input.indexOf("?"); + if (qIndex === -1) return null; + return new URLSearchParams(input.slice(qIndex + 1)); + } +} + +function read( + search: URLSearchParams, + key: string +): string | null { + const value = search.get(key)?.trim(); + if (!value) return null; + return value.slice(0, MAX_LEN); +} + +/** + * Extracts UTM parameters from any number of URL-ish strings. + * Earlier arguments win per key; later ones only fill gaps. + * + * @example + * parseUtm("https://nook.freno.me/?utm_source=hn&utm_campaign=launch") + * // { utmSource: "hn", utmMedium: null, utmCampaign: "launch" } + */ +export function parseUtm( + ...inputs: (string | null | undefined)[] +): UtmParams { + const result: UtmParams = { ...EMPTY }; + + for (const input of inputs) { + if (!input) continue; + // Everything present already — nothing left for later inputs to fill. + if (result.utmSource && result.utmMedium && result.utmCampaign) break; + + const search = searchOf(input); + if (!search) continue; + + result.utmSource ||= read(search, "utm_source"); + result.utmMedium ||= read(search, "utm_medium"); + result.utmCampaign ||= read(search, "utm_campaign"); + } + + return result; +}