Mike/freno-dev
1
0
Fork 3
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

analytics: make each weekly funnel stage separately measurable (FRE-366)

Browse Source 
Gap 1 — click and served share one label: trackDownload now takes a
stage ("click" | "served") carried on a new VisitorAnalytics.stage
column; the Nook door and the tRPC proxy record clicks, the DMG file
route records served. summariseDownloads() and the /analytics downloads
block are untouched — the stage is a separate column, so the path
grouping cannot change.

Gap 2 — checkout start records nothing: the checkout route records a
created Stripe session through the shared trackRequest writer (same
referrer, campaign, country dimensions as every door) under its own
path, and STAGES.checkout reads it instead of the /checkout page proxy.

Gap 3 — loopback addresses, no host dimension: every writer now
resolves the client address through resolveClientIp, which trusts
x-forwarded-for only when the direct peer is the loopback production
proxy (trust boundary stated in request-client.ts), and records the
host on each row through the same site-context extractor routing uses.
stage and host reach the table through VISITOR_ANALYTICS_ADDED_COLUMNS;
the weekly job applies the same bootstrap before reading so it renders
seven stages against a database no deploy has migrated yet.

The weekly read splits downloadClick/dmgServed, scopes visits to
nook.freno.me, and computes its caveats from the window: pre-instrumentation
rows are stated and excluded, and the loopback warning now fires only
when rows recorded since the instrumentation still carry it.
This commit is contained in:
Paperclip Agent
2026-10-29 17:02:39 +00:00
parent 89f56fc78a
commit 39d874a7e2
19 changed files with 643 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
src/server/weekly-metrics.test.ts
View File

@@ -1,12 +1,16 @@
import { beforeEach, describe, expect, it } from "bun:test";
import { Database } from "bun:sqlite";
import {
VISITOR_ANALYTICS_ADDED_COLUMNS,
VISITOR_ANALYTICS_COLUMNS,
VISITOR_ANALYTICS_DDL,
VISITOR_ANALYTICS_INSERT
} from "./analytics-schema";
import {
NOOK_CHECKOUT_PATH,
NOOK_DOWNLOAD_PATH,
NOOK_HOST,
ensureAnalyticsSchema,
getWeeklyReport,
pickMover,
readCampaigns,
@@ -70,6 +74,8 @@ let conns: Connections;
const NOW = new Date("2026-10-29T16:00:00Z");
// A row as the post-FRE-366 writers record it: real client address, the host
// it addressed, and a stage label where the row is a download.
const row = (
overrides: Record<string, unknown>
): Record<string, unknown> => ({
@@ -96,6 +102,8 @@ const row = (
ttfb: null,
dom_load: null,
load_complete: null,
stage: null,
host: NOOK_HOST,
...overrides
});
@@ -142,20 +150,32 @@ describe("windowsFor", () => {
});
describe("readFunnel", () => {
it("counts each stage inside its window and nothing outside it", async () => {
// Page views count; API calls do not.
it("counts each stage by its own label inside its window", async () => {
// Page views on the Nook host count; other sites, API calls, legacy
// rows with no host, and the other window do not.
insertAnalytics({ path: "/", created_at: THIS_WEEK });
insertAnalytics({ path: "/downloads", created_at: THIS_WEEK });
insertAnalytics({ path: "/", host: "freno.me", created_at: THIS_WEEK });
insertAnalytics({ path: "/", host: null, created_at: THIS_WEEK });
insertAnalytics({ path: "/api/trpc/anything", created_at: THIS_WEEK });
insertAnalytics({ path: "/", created_at: LAST_WEEK });
// The merged download label, click and served alike.
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, created_at: THIS_WEEK });
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, created_at: THIS_WEEK });
// Checkout page doors, including the legacy /nook/checkout path.
// The old checkout proxy: a /checkout page view is a visit, not a
// started checkout.
insertAnalytics({ path: "/checkout", created_at: THIS_WEEK });
insertAnalytics({ path: "/nook/checkout", created_at: THIS_WEEK });
// The download label splits by stage; a row with no stage predates the
// split and counts in neither.
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, stage: "click", created_at: THIS_WEEK });
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, stage: "served", created_at: THIS_WEEK });
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, stage: "served", created_at: THIS_WEEK });
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, stage: null, created_at: THIS_WEEK });
// A created Stripe session, recorded by the checkout route itself.
insertAnalytics({
path: NOOK_CHECKOUT_PATH,
method: "POST",
created_at: THIS_WEEK
});
// The licence DB stores ISO-8601 with milliseconds.
nookDb
@@ -180,13 +200,12 @@ describe("readFunnel", () => {
const window = windowsFor(NOW, 7).current;
const funnel = await readFunnel(conns, window);
// "/checkout" and "/nook/checkout" are page views too, so they land in
// both the visit count and the checkout count.
expect(funnel).toEqual({
visit: 4,
download: 2,
visit: 3, // "/", "/downloads" and the "/checkout" page view on the Nook host
downloadClick: 1,
dmgServed: 2,
trial: 1,
checkout: 2,
checkout: 1,
purchase: 1,
activation: 1
});
@@ -196,7 +215,8 @@ describe("readFunnel", () => {
const window = windowsFor(NOW, 7).current;
expect(await readFunnel(conns, window)).toEqual({
visit: 0,
download: 0,
downloadClick: 0,
dmgServed: 0,
trial: 0,
checkout: 0,
purchase: 0,
@@ -206,37 +226,51 @@ describe("readFunnel", () => {
});
describe("readTraffic", () => {
it("separates loopback and automation from browser traffic", async () => {
it("separates legacy rows, live loopback and automation from clean traffic", async () => {
// Post-instrumentation rows: host set. One of them still loopback — the
// shape the report must flag as "the fix is not live".
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
stage: "click",
created_at: THIS_WEEK
});
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
stage: "served",
ip_address: "127.0.0.1",
user_agent: "The Nook/0.13.4 Sparkle/2.9.6",
created_at: THIS_WEEK
});
insertAnalytics({ path: "/", created_at: THIS_WEEK });
// Pre-instrumentation rows: no host, proxy's address — legacy, never
// counted as "the fix is not live".
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
stage: null,
host: null,
ip_address: "127.0.0.1",
user_agent: "curl/8.7.1",
created_at: THIS_WEEK
});
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
path: "/",
host: null,
ip_address: "::1",
user_agent: "The Nook/0.13.4 Sparkle/2.9.6",
created_at: THIS_WEEK
});
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
ip_address: "203.0.113.9",
user_agent: "Mozilla/5.0 (Macintosh) Safari/605.1.15",
created_at: THIS_WEEK
});
insertAnalytics({ path: "/", ip_address: "203.0.113.9", created_at: THIS_WEEK });
const traffic = await readTraffic(conns, windowsFor(NOW, 7).current);
expect(traffic).toEqual({
analyticsRows: 4,
loopbackRows: 2,
allTimeRows: 4,
allTimeAddressedRows: 4,
allTimeLoopbackRows: 2,
analyticsRows: 5,
hostRows: 3,
legacyRows: 2,
hostLoopbackRows: 1,
downloadRows: 3,
downloadLoopbackRows: 2,
downloadClickRows: 1,
downloadServedRows: 1,
downloadUnstagedRows: 1,
downloadNonBrowserRows: 2
});
});
@@ -288,10 +322,53 @@ describe("readCampaigns", () => {
});
});
describe("ensureAnalyticsSchema", () => {
it("adds the post-hoc columns to a table that predates them, idempotently", async () => {
// The table as production held it before any post-hoc column existed.
// (`created_at` is not in the insert column list — it defaults DB-side —
// so the simulated table has to name it explicitly.)
const oldDb = new Database(":memory:");
const withoutAdded = [
...VISITOR_ANALYTICS_COLUMNS.filter(
(c) => !VISITOR_ANALYTICS_ADDED_COLUMNS.some((a) => a.name === c)
),
"created_at"
];
oldDb.run(
`CREATE TABLE VisitorAnalytics (${withoutAdded.map((c) => `${c} TEXT`).join(", ")})`
);
const oldConn = adapter(oldDb);
await ensureAnalyticsSchema(oldConn);
await ensureAnalyticsSchema(oldConn); // a second run is a no-op
// The writers' full INSERT — stage and host included — now works.
const values = row({ stage: "click", path: NOOK_DOWNLOAD_PATH });
expect(() =>
oldDb
.prepare(VISITOR_ANALYTICS_INSERT)
.run(
...(VISITOR_ANALYTICS_COLUMNS.map((c) => values[c]) as never[])
)
).not.toThrow();
oldDb
.prepare(`UPDATE VisitorAnalytics SET created_at = ? WHERE id = ?`)
.run(THIS_WEEK, values.id as string);
// And the funnel reads its stage split from the migrated table.
const funnel = await readFunnel(
{ analytics: oldConn, nook: conns.nook },
windowsFor(NOW, 7).current
);
expect(funnel.downloadClick).toBe(1);
});
});
describe("pickMover", () => {
const base: FunnelCounts = {
visit: 0,
download: 0,
downloadClick: 0,
dmgServed: 0,
trial: 0,
checkout: 0,
purchase: 0,
@@ -300,12 +377,12 @@ describe("pickMover", () => {
it("picks the stage with the largest absolute change", () => {
const mover = pickMover(
{ ...base, visit: 61, download: 156, trial: 2 },
{ ...base, visit: 61, dmgServed: 156, trial: 2 },
{ ...base, visit: 56 }
);
expect(mover).toEqual({
key: "download",
label: "download click → DMG served (one label)",
key: "dmgServed",
label: "DMG served",
now: 156,
before: 0,
delta: 156
@@ -314,8 +391,8 @@ describe("pickMover", () => {
it("breaks ties toward the earlier stage", () => {
const mover = pickMover(
{ ...base, visit: 5, download: 5 },
{ ...base, visit: 0, download: 0 }
{ ...base, visit: 5, downloadClick: 5 },
{ ...base, visit: 0, downloadClick: 0 }
);
expect(mover?.key).toBe("visit");
});
@@ -326,20 +403,20 @@ describe("pickMover", () => {
});
describe("getWeeklyReport / renderWeeklyMarkdown", () => {
it("reports a week in the agreed stage order with the computed caveats", async () => {
it("renders seven distinct stages with no merged-label or proxy caveats", async () => {
insertAnalytics({ path: "/", created_at: THIS_WEEK });
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, stage: "click", created_at: THIS_WEEK });
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
ip_address: "127.0.0.1",
user_agent: "curl/8.7.1",
stage: "served",
created_at: THIS_WEEK
});
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
ip_address: "127.0.0.1",
user_agent: "curl/8.7.1",
stage: "served",
created_at: THIS_WEEK
});
insertAnalytics({ path: NOOK_CHECKOUT_PATH, method: "POST", created_at: THIS_WEEK });
nookDb
.prepare(`INSERT INTO trials (fingerprint, started_at) VALUES (?, ?)`)
.run("fp-1", "2026-10-28T20:28:46.806Z");
@@ -347,21 +424,72 @@ describe("getWeeklyReport / renderWeeklyMarkdown", () => {
const report = await getWeeklyReport(conns, { now: NOW, days: 7 });
const markdown = renderWeeklyMarkdown(report);
expect(report.mover?.key).toBe("download");
expect(markdown).toContain(
"Window: **2026-10-22 16:00:00 → 2026-10-29 16:00:00 UTC** (7 days)"
);
expect(markdown).toContain("| visit (page views) | 1 | 0 | +1 |");
expect(markdown.indexOf("visit (page views)")).toBeLessThan(
markdown.indexOf("download click → DMG served")
expect(markdown).toContain(`| visit (${NOOK_HOST} page views) | 1 | 0 | +1 |`);
expect(markdown).toContain("| download click | 1 | 0 | +1 |");
expect(markdown).toContain("| DMG served | 2 | 0 | +2 |");
expect(markdown).toContain("| checkout started (Stripe session) | 1 | 0 | +1 |");
const order = [
`visit (${NOOK_HOST} page views)`,
"download click",
"DMG served",
"trial started",
"checkout started (Stripe session)",
"purchase (licence issued)",
"activation"
];
for (let i = 1; i < order.length; i += 1) {
expect(markdown.indexOf(order[i - 1]!)).toBeLessThan(
markdown.indexOf(order[i]!)
);
}
// The caveats this issue retired stay retired.
expect(markdown).not.toContain("one label");
expect(markdown).not.toContain("not recorded");
expect(markdown).not.toContain("no host dimension");
expect(markdown).not.toContain("†");
expect(report.mover?.key).toBe("dmgServed");
expect(markdown).toContain("**DMG served: 0 → 2 (+2).**");
});
it("states what pre-instrumentation rows cannot say, computed from the window", async () => {
insertAnalytics({ path: "/", created_at: THIS_WEEK });
// Legacy rows: no host, proxy's address, no stage label.
insertAnalytics({ path: "/", host: null, ip_address: "127.0.0.1", created_at: THIS_WEEK });
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
stage: null,
host: null,
ip_address: "127.0.0.1",
user_agent: "curl/8.7.1",
created_at: THIS_WEEK
});
const report = await getWeeklyReport(conns, { now: NOW, days: 7 });
const markdown = renderWeeklyMarkdown(report);
expect(markdown).toContain(
"2 of 3 rows in the window predate the FRE-366 instrumentation"
);
expect(markdown.indexOf("download click → DMG served")).toBeLessThan(
markdown.indexOf("trial started")
expect(markdown).toContain("predate the click/served stage label");
// Legacy loopback is not "the fix is not live".
expect(markdown).not.toContain("still carry a loopback address");
});
it("flags loopback only when rows recorded since the instrumentation carry it", async () => {
insertAnalytics({ path: "/", ip_address: "127.0.0.1", created_at: THIS_WEEK });
const report = await getWeeklyReport(conns, { now: NOW, days: 7 });
const markdown = renderWeeklyMarkdown(report);
expect(markdown).toContain(
"1 rows recorded since the instrumentation still carry a loopback address"
);
expect(markdown).toContain("**download click → DMG served (one label): 0 → 2 (+2).**");
expect(markdown).toContain("2 of 3 rows in the window came from a loopback address");
expect(markdown).toContain("Checkout start is not recorded");
expect(markdown).toContain("There is no host dimension");
});
it("says nothing moved rather than inventing a winner", async () => {