agent-security-engineer/freno-dev
1
0
Fork 0
forked from Mike/freno-dev
Code Pull Requests Activity

nook marketing - utm, analytics, etc

Browse Source
This commit is contained in:
Michael Freno
2026-10-03 11:03:26 -04:00
parent 3fe3516b9e
commit 96e401ecbb
11 changed files with 911 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
src/server/analytics-schema.test.ts Normal file
View File

@@ -0,0 +1,137 @@
import { describe, expect, it, beforeEach } from "bun:test";
import { Database } from "bun:sqlite";
import {
VISITOR_ANALYTICS_ADDED_COLUMNS,
VISITOR_ANALYTICS_COLUMNS,
VISITOR_ANALYTICS_DDL,
VISITOR_ANALYTICS_INSERT,
VISITOR_ANALYTICS_INDEXES
} from "./analytics-schema";
// Why an in-memory SQLite database: VisitorAnalytics lived only in the live
// Turso database, so nothing in the repo proved the DDL and the INSERT
// agreed. These tests apply the real DDL to a throwaway database and run the
// real INSERT statement against it — the exact class of drift (a column in
// the INSERT that the table does not have) fails here instead of silently in
// production, where the buffer swallows write errors.
let db: Database;
const sampleArgs = [
"row-1", // id
null, // user_id
"/nook/", // path
"GET", // method
"https://nook.freno.me/", // referrer
"Mozilla/5.0 (Macintosh)", // user_agent
"127.0.0.1", // ip_address
"GB", // country
"desktop", // device_type
"safari", // browser
"macos", // os
"hn", // utm_source
"post", // utm_medium
"launch", // utm_campaign
12, // duration_ms
100, // fcp
200, // lcp
0.01, // cls
1, // fid
2, // inp
3, // ttfb
4, // dom_load
5 // load_complete
];
beforeEach(() => {
db = new Database(":memory:");
db.run(VISITOR_ANALYTICS_DDL);
for (const index of VISITOR_ANALYTICS_INDEXES) db.run(index);
});
describe("VisitorAnalytics schema", () => {
it("creates the table idempotently", () => {
expect(() => db.run(VISITOR_ANALYTICS_DDL)).not.toThrow();
expect(() => {
for (const index of VISITOR_ANALYTICS_INDEXES) db.run(index);
}).not.toThrow();
});
it("defines every column the writers insert", () => {
const columns = (
db.prepare(`PRAGMA table_info(VisitorAnalytics)`).all() as {
name: string;
}[]
).map((c) => c.name);
for (const column of VISITOR_ANALYTICS_COLUMNS) {
expect(columns).toContain(column);
}
// The reverse direction matters too: a column the code writes but the
// DDL omits is the failure this whole file exists to catch.
expect(VISITOR_ANALYTICS_COLUMNS.length).toBe(23);
});
it("has one placeholder per inserted column", () => {
const placeholders = (VISITOR_ANALYTICS_INSERT.match(/\?/g) ?? []).length;
expect(placeholders).toBe(VISITOR_ANALYTICS_COLUMNS.length);
expect(sampleArgs.length).toBe(VISITOR_ANALYTICS_COLUMNS.length);
});
it("accepts a full row and returns the campaign columns", () => {
db.prepare(VISITOR_ANALYTICS_INSERT).run(...sampleArgs);
const row = db
.prepare(
`SELECT path, country, utm_source, utm_medium, utm_campaign
FROM VisitorAnalytics`
)
.get() as Record<string, unknown>;
expect(row).toEqual({
path: "/nook/",
country: "GB",
utm_source: "hn",
utm_medium: "post",
utm_campaign: "launch"
});
});
it("accepts a row with no campaign tags", () => {
const args = [...sampleArgs];
args[11] = args[12] = args[13] = null;
db.prepare(VISITOR_ANALYTICS_INSERT).run(...args);
const row = db
.prepare(`SELECT utm_source FROM VisitorAnalytics`)
.get() as Record<string, unknown>;
expect(row.utm_source).toBeNull();
});
it("adds the post-hoc columns to a table that predates them", () => {
// Simulate the older table: create it without the utm columns, then run
// the ALTER path the bootstrap uses and confirm the writers still work.
const old = new Database(":memory:");
const withoutUtm = VISITOR_ANALYTICS_COLUMNS.filter(
(c) => !VISITOR_ANALYTICS_ADDED_COLUMNS.some((a) => a.name === c)
);
old.run(
`CREATE TABLE VisitorAnalytics (${withoutUtm
.map((c) => `${c} TEXT`)
.join(", ")})`
);
const present = new Set(
(old.prepare(`PRAGMA table_info(VisitorAnalytics)`).all() as {
name: string;
}[]).map((c) => c.name)
);
for (const column of VISITOR_ANALYTICS_ADDED_COLUMNS) {
if (!present.has(column.name)) old.run(column.ddl);
}
expect(() =>
old.prepare(VISITOR_ANALYTICS_INSERT).run(...sampleArgs)
).not.toThrow();
});
});

123
src/server/analytics-schema.ts Normal file
View File

@@ -0,0 +1,123 @@
/**
* VisitorAnalytics schema — the single source of truth for the table, its
* indexes, and the column list both writers insert.
*
* Why this file exists: `VisitorAnalytics` had no `CREATE TABLE` anywhere in
* this repo. It existed only in the live Turso database, so a fresh or
* restored environment had nothing to write into — and because
* `flushAnalyticsBuffer` swallows errors by design, every insert would fail
* silently. The definition below mirrors production exactly.
*
* `session_id` is deliberately kept. It is a leftover from the removed
* analytics-page feature (added in d2ee61b, dropped from the code in
* 58d48da) — nothing reads or writes it now, and the Nook macOS app never
* touches this table at all. SQLite would let us `DROP COLUMN`, but the
* column costs nothing and the live data in it may still be wanted, so the
* schema records it rather than removing it.
*
* Pure string constants, no imports — so the parity test can apply them to
* an in-memory SQLite database without a network or environment.
*/
/**
* The table as it exists in production. `IF NOT EXISTS` makes applying this
* a no-op against the live database; it only does work on a database that
* has never had the table.
*/
export const VISITOR_ANALYTICS_DDL = `
CREATE TABLE IF NOT EXISTS VisitorAnalytics (
id TEXT PRIMARY KEY,
user_id TEXT,
path TEXT NOT NULL,
method TEXT NOT NULL,
referrer TEXT,
user_agent TEXT,
ip_address TEXT,
country TEXT,
device_type TEXT,
browser TEXT,
os TEXT,
session_id TEXT,
duration_ms INTEGER,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
fcp REAL,
lcp REAL,
cls REAL,
fid REAL,
inp REAL,
ttfb REAL,
dom_load REAL,
load_complete REAL,
utm_source TEXT,
utm_medium TEXT,
utm_campaign TEXT,
FOREIGN KEY (user_id) REFERENCES User(id) ON DELETE SET NULL
)
`;
/** Indexes the summary, funnel and path queries rely on. */
export const VISITOR_ANALYTICS_INDEXES = [
`CREATE INDEX IF NOT EXISTS idx_analytics_user_id ON VisitorAnalytics(user_id)`,
`CREATE INDEX IF NOT EXISTS idx_analytics_created_at ON VisitorAnalytics(created_at)`,
`CREATE INDEX IF NOT EXISTS idx_analytics_path ON VisitorAnalytics(path)`,
`CREATE INDEX IF NOT EXISTS idx_analytics_ip_address ON VisitorAnalytics(ip_address)`,
`CREATE INDEX IF NOT EXISTS idx_analytics_session_id ON VisitorAnalytics(session_id)`,
`CREATE INDEX IF NOT EXISTS idx_analytics_lcp ON VisitorAnalytics(lcp) WHERE lcp IS NOT NULL`,
`CREATE INDEX IF NOT EXISTS idx_analytics_fcp ON VisitorAnalytics(fcp) WHERE fcp IS NOT NULL`
];
/** Columns that may be missing from a table created before they existed. */
export const VISITOR_ANALYTICS_ADDED_COLUMNS = [
{
name: "utm_source",
ddl: `ALTER TABLE VisitorAnalytics ADD COLUMN utm_source TEXT`
},
{
name: "utm_medium",
ddl: `ALTER TABLE VisitorAnalytics ADD COLUMN utm_medium TEXT`
},
{
name: "utm_campaign",
ddl: `ALTER TABLE VisitorAnalytics ADD COLUMN utm_campaign TEXT`
}
];
/**
* Every column both writers insert, in order. The parity test asserts these
* all exist on the table, which is what stops the DDL and the INSERT from
* drifting apart again.
*/
export const VISITOR_ANALYTICS_COLUMNS = [
"id",
"user_id",
"path",
"method",
"referrer",
"user_agent",
"ip_address",
"country",
"device_type",
"browser",
"os",
"utm_source",
"utm_medium",
"utm_campaign",
"duration_ms",
"fcp",
"lcp",
"cls",
"fid",
"inp",
"ttfb",
"dom_load",
"load_complete"
] as const;
/**
* The one INSERT statement used by both analytics writers — the buffered
* writer in `analytics.ts` and the page-visit path in the tRPC router.
* Shared so the two can never disagree about column order or count.
*/
export const VISITOR_ANALYTICS_INSERT = `INSERT INTO VisitorAnalytics (
${VISITOR_ANALYTICS_COLUMNS.join(", ")}
) VALUES (${VISITOR_ANALYTICS_COLUMNS.map(() => "?").join(", ")})`;

98
src/server/analytics.ts
View File

@@ -2,6 +2,43 @@ import { ConnectionFactory } from "./database";
import { v4 as uuid } from "uuid";
import type { VisitorAnalytics, AnalyticsQuery } from "~/db/types";
import { CACHE_CONFIG } from "~/config";
import {
VISITOR_ANALYTICS_ADDED_COLUMNS,
VISITOR_ANALYTICS_DDL,
VISITOR_ANALYTICS_INDEXES,
VISITOR_ANALYTICS_INSERT
} from "./analytics-schema";
/**
* Idempotent schema bootstrap, mirroring `nook.ts`. The table had no DDL in
* this repo, so a database that never had it would accept no writes at all
* (and the buffer swallows its errors, so the failure would be invisible).
* Against the live database this is a no-op; it only creates what is absent.
*/
export const analyticsSchemaBootstrap: Promise<unknown> = (async () => {
const conn = ConnectionFactory();
await conn.execute(VISITOR_ANALYTICS_DDL);
for (const index of VISITOR_ANALYTICS_INDEXES) {
await conn.execute(index);
}
// SQLite has no "ADD COLUMN IF NOT EXISTS", so check before altering —
// covers a database restored from before these columns existed.
const info = await conn.execute(`PRAGMA table_info(VisitorAnalytics)`);
const present = new Set(info.rows.map((row) => String((row as any).name)));
for (const column of VISITOR_ANALYTICS_ADDED_COLUMNS) {
if (!present.has(column.name)) {
await conn.execute(column.ddl);
}
}
})();
// A bootstrap failure must not become an unhandled rejection on a request
// path that merely imports this module.
analyticsSchemaBootstrap.catch((error) =>
console.error("VisitorAnalytics schema bootstrap failed:", error)
);
export interface AnalyticsEntry {
userId?: string | null;
@@ -14,6 +51,9 @@ export interface AnalyticsEntry {
deviceType?: string | null;
browser?: string | null;
os?: string | null;
utmSource?: string | null;
utmMedium?: string | null;
utmCampaign?: string | null;
durationMs?: number | null;
fcp?: number | null;
lcp?: number | null;
@@ -59,11 +99,7 @@ async function flushAnalyticsBuffer(): Promise<void> {
// Batch insert - more efficient than individual inserts
for (const entry of entriesToWrite) {
await conn.execute({
sql: `INSERT INTO VisitorAnalytics (
id, user_id, path, method, referrer, user_agent, ip_address,
country, device_type, browser, os, duration_ms,
fcp, lcp, cls, fid, inp, ttfb, dom_load, load_complete
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
sql: VISITOR_ANALYTICS_INSERT,
args: [
uuid(),
entry.userId || null,
@@ -76,6 +112,9 @@ async function flushAnalyticsBuffer(): Promise<void> {
entry.deviceType || null,
entry.browser || null,
entry.os || null,
entry.utmSource || null,
entry.utmMedium || null,
entry.utmCampaign || null,
entry.durationMs || null,
entry.fcp || null,
entry.lcp || null,
@@ -94,6 +133,23 @@ async function flushAnalyticsBuffer(): Promise<void> {
}
}
/**
* Write buffered entries now instead of waiting for the batch timer.
*
* The buffer only writes immediately at ANALYTICS_BATCH_SIZE entries; below
* that it defers to a 30-second timeout. On a long-lived server that is
* fine, but on serverless the work scheduled after a response is not
* guaranteed to run before the instance is recycled — so a low-volume but
* high-value event (a download click) could sit in memory and never be
* recorded. Callers that must not lose a row await this.
*
* Safe to call concurrently: the buffer is drained synchronously before the
* first await, so entries are never written twice.
*/
export async function flushAnalytics(): Promise<void> {
await flushAnalyticsBuffer();
}
/**
* Schedule periodic buffer flush
*/
@@ -197,6 +253,9 @@ export async function queryAnalytics(
device_type: row.device_type as string | null,
browser: row.browser as string | null,
os: row.os as string | null,
utm_source: row.utm_source as string | null,
utm_medium: row.utm_medium as string | null,
utm_campaign: row.utm_campaign as string | null,
duration_ms: row.duration_ms as number | null,
created_at: row.created_at as string
}));
@@ -213,6 +272,12 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
topReferrers: Array<{ referrer: string; count: number }>;
deviceTypes: Array<{ type: string; count: number }>;
browsers: Array<{ browser: string; count: number }>;
topCampaigns: Array<{
source: string | null;
medium: string | null;
campaign: string | null;
count: number;
}>;
}> {
const conn = ConnectionFactory();
@@ -324,6 +389,26 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
count: row.count as number
}));
// Campaign attribution (§8): which channel produced the traffic. Rows with
// no tags are excluded — the "untagged" bucket is a separate question and
// would otherwise dominate the list.
const topCampaignsResult = await conn.execute({
sql: `SELECT utm_source, utm_medium, utm_campaign, COUNT(*) as count
FROM VisitorAnalytics
WHERE created_at >= datetime('now', '-${days} days')
AND (utm_source IS NOT NULL OR utm_medium IS NOT NULL OR utm_campaign IS NOT NULL)
GROUP BY utm_source, utm_medium, utm_campaign
ORDER BY count DESC
LIMIT 10`,
args: []
});
const topCampaigns = topCampaignsResult.rows.map((row) => ({
source: (row.utm_source as string | null) ?? null,
medium: (row.utm_medium as string | null) ?? null,
campaign: (row.utm_campaign as string | null) ?? null,
count: row.count as number
}));
return {
totalVisits,
totalPageVisits,
@@ -334,7 +419,8 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
topApiCalls,
topReferrers,
deviceTypes,
browsers
browsers,
topCampaigns
};
}

15
src/server/api/routers/analytics.ts
View File

@@ -11,6 +11,8 @@ import {
import { ConnectionFactory } from "~/server/database";
import { v4 as uuid } from "uuid";
import { getRequestIP } from "vinxi/http";
import { parseUtm } from "~/server/utm";
import { VISITOR_ANALYTICS_INSERT } from "~/server/analytics-schema";
/** Safely get a header value from either Fetch API Headers or Node.js IncomingHttpHeaders */
function getHeader(
@@ -104,6 +106,10 @@ export const analyticsRouter = createTRPCRouter({
userAgent,
referrer,
ipAddress,
// The client beacon sends `pathname + search`, so the page URL's
// own query string is the campaign source; the referrer fills any
// gap (e.g. clicks that arrive on a clean path).
...parseUtm(input.path, referrer),
fcp: input.metrics.fcp,
lcp: input.metrics.lcp,
cls: input.metrics.cls,
@@ -115,11 +121,7 @@ export const analyticsRouter = createTRPCRouter({
});
await conn.execute({
sql: `INSERT INTO VisitorAnalytics (
id, user_id, path, method, referrer, user_agent, ip_address,
country, device_type, browser, os, duration_ms,
fcp, lcp, cls, fid, inp, ttfb, dom_load, load_complete
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
sql: VISITOR_ANALYTICS_INSERT,
args: [
uuid(),
enriched.userId || null,
@@ -132,6 +134,9 @@ export const analyticsRouter = createTRPCRouter({
enriched.deviceType || null,
enriched.browser || null,
enriched.os || null,
enriched.utmSource || null,
enriched.utmMedium || null,
enriched.utmCampaign || null,
enriched.durationMs || null,
enriched.fcp || null,
enriched.lcp || null,

85
src/server/utm.test.ts Normal file
View File

@@ -0,0 +1,85 @@
import { describe, expect, it } from "bun:test";
import { parseUtm } from "./utm";
describe("parseUtm", () => {
it("reads the three keys from a full URL", () => {
expect(
parseUtm(
"https://nook.freno.me/?utm_source=hn&utm_medium=post&utm_campaign=launch"
)
).toEqual({
utmSource: "hn",
utmMedium: "post",
utmCampaign: "launch"
});
});
it("reads a relative path with a query (the client beacon's shape)", () => {
expect(parseUtm("/nook/?utm_source=r_macapps")).toEqual({
utmSource: "r_macapps",
utmMedium: null,
utmCampaign: null
});
});
it("returns all nulls when nothing is tagged", () => {
const empty = { utmSource: null, utmMedium: null, utmCampaign: null };
expect(parseUtm("https://nook.freno.me/")).toEqual(empty);
expect(parseUtm("/nook/?ref=friend")).toEqual(empty);
expect(parseUtm(null)).toEqual(empty);
expect(parseUtm(undefined)).toEqual(empty);
expect(parseUtm("")).toEqual(empty);
});
it("lets an earlier argument win per key, later ones filling gaps", () => {
// A download click with no query of its own takes all three from the
// landing page carried in the referer.
expect(
parseUtm(
"https://nook.freno.me/api/the-nook/download",
"https://nook.freno.me/?utm_source=hn&utm_campaign=launch"
)
).toEqual({
utmSource: "hn",
utmMedium: null,
utmCampaign: "launch"
});
// The click's own tags win; the referer only supplies what is missing.
expect(
parseUtm(
"https://nook.freno.me/api/the-nook/download?utm_source=x",
"https://nook.freno.me/?utm_source=hn&utm_campaign=launch"
)
).toEqual({
utmSource: "x",
utmMedium: null,
utmCampaign: "launch"
});
});
it("trims whitespace, drops empties, and caps length", () => {
expect(parseUtm("https://x.test/?utm_source=%20hn%20&utm_medium=")).toEqual({
utmSource: "hn",
utmMedium: null,
utmCampaign: null
});
const long = parseUtm(
`https://x.test/?utm_campaign=${"a".repeat(500)}`
);
expect(long.utmCampaign!.length).toBe(200);
});
it("does not case-fold values", () => {
expect(parseUtm("https://x.test/?utm_campaign=Launch").utmCampaign).toBe(
"Launch"
);
});
it("ignores unrelated parameters", () => {
expect(
parseUtm("https://x.test/?utm_source=hn&c=GB&gclid=abc")
).toEqual({ utmSource: "hn", utmMedium: null, utmCampaign: null });
});
});

82
src/server/utm.ts Normal file
View File

@@ -0,0 +1,82 @@
/**
* Campaign parameters, parsed once and shared by every analytics writer.
*
* The three keys kept here are the ones a launch plan actually groups by:
* which channel (`utm_source`), through what (`utm_medium`), and which
* specific push (`utm_campaign`). Values are trimmed and length-capped,
* but never case-folded — an uppercase campaign name is a different
* campaign until the author says otherwise.
*
* `parseUtm` is variadic and first-non-empty-wins per key, which is what
* lets a logger pass both the URL it was called on and the referrer: a
* download click carries no query of its own, but the landing page it
* came from usually still does.
*/
export interface UtmParams {
utmSource: string | null;
utmMedium: string | null;
utmCampaign: string | null;
}
const EMPTY: UtmParams = {
utmSource: null,
utmMedium: null,
utmCampaign: null
};
/** Longest value we will store; far above any real campaign name. */
const MAX_LEN = 200;
/**
* Reads the query string out of a full URL, a path with a query, or a bare
* `a=1&b=2` fragment. Returns null when there is nothing to read.
*/
function searchOf(input: string): URLSearchParams | null {
try {
return new URL(input).searchParams;
} catch {
// Not absolute — take everything after the first "?".
const qIndex = input.indexOf("?");
if (qIndex === -1) return null;
return new URLSearchParams(input.slice(qIndex + 1));
}
}
function read(
search: URLSearchParams,
key: string
): string | null {
const value = search.get(key)?.trim();
if (!value) return null;
return value.slice(0, MAX_LEN);
}
/**
* Extracts UTM parameters from any number of URL-ish strings.
* Earlier arguments win per key; later ones only fill gaps.
*
* @example
* parseUtm("https://nook.freno.me/?utm_source=hn&utm_campaign=launch")
* // { utmSource: "hn", utmMedium: null, utmCampaign: "launch" }
*/
export function parseUtm(
...inputs: (string | null | undefined)[]
): UtmParams {
const result: UtmParams = { ...EMPTY };
for (const input of inputs) {
if (!input) continue;
// Everything present already — nothing left for later inputs to fill.
if (result.utmSource && result.utmMedium && result.utmCampaign) break;
const search = searchOf(input);
if (!search) continue;
result.utmSource ||= read(search, "utm_source");
result.utmMedium ||= read(search, "utm_medium");
result.utmCampaign ||= read(search, "utm_campaign");
}
return result;
}