agent-security-engineer/freno-dev
1
0
Fork 0
forked from Mike/freno-dev
Code Pull Requests Activity

metrics: stand up the weekly funnel job (FRE-202)

Browse Source 
`marketing/metrics/weekly.md` has been a placeholder since the marketing tree
landed — it promised a funnel and the one thing that moved, and nothing
produced it. This adds the producer: one command reads the live databases and
rewrites the file.

`scripts/weekly-metrics.ts` writes `marketing/metrics/weekly.md` from
`VisitorAnalytics` (visits, downloads, campaigns) and the-nook's `trials`,
`licenses`, `activations` (the paid funnel). It needs four env vars —
TURSO_DB_URL/TOKEN and NOOK_DB_URL/TOKEN — and imports no server env, so the
job runs from a checkout that has no Stripe or Apple secrets.
`src/server/weekly-metrics.ts` holds the queries and the renderer, taking both
connections as arguments; the tests run them against an in-memory SQLite shaped
like production, the same way the backfill suite does.

The campaign breakdown groups exactly as `getAnalyticsSummary.topCampaigns`
does (top 10, untagged excluded), so the weekly read and /analytics cannot
drift into different label sets.

The first real report says more about the instrumentation than about demand,
and the renderer computes that rather than asserting it:

- every addressed row the table has ever held (12,035 of 12,048) is loopback,
  so "unique visitors" is one host and per-channel reach cannot be read;
- the landing door and the file route both write `/api/downloads/thenook`, so
  "download click" and "DMG served" are one label — one served click writes two
  rows, a failed click one, a Sparkle update one;
- `POST /api/the-nook/checkout` records nothing, so checkout start is only a
  page-view proxy;
- `VisitorAnalytics` has no host dimension, so the Nook's `/` is freno.me's `/`.

Those three gaps are reported beside the numbers, not papered over. Week 0
(2026-10-22 → 2026-10-29): visit 61, download 156, trial 2, checkout 0,
purchase 0, activation 0 — and the one thing that moved is the download stage,
0 → 156, because the label was introduced that week.

Tests: `bun test src/server/weekly-metrics.test.ts` — 12 pass, 0 fail. The full
suite's 15 failures are pre-existing: this checkout's .env lacks
NOOK_LICENSE_PRIVATE_KEY, NOOK_STRIPE_SK/WEBHOOK/PRICE and LINEAGE_JWT_SECRET,
and every failure is that env validation.

Scripts/ is gitignored, so `scripts/weekly-metrics.ts` was added with `-f`, the
way the existing tracked scripts are.

FRE-202
This commit is contained in:
Paperclip Agent
2026-10-29 15:58:51 +00:00
parent d44133fcb3
commit 89f56fc78a
6 changed files with 1184 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

381
src/server/weekly-metrics.test.ts Normal file
View File

@@ -0,0 +1,381 @@
import { beforeEach, describe, expect, it } from "bun:test";
import { Database } from "bun:sqlite";
import {
VISITOR_ANALYTICS_COLUMNS,
VISITOR_ANALYTICS_DDL,
VISITOR_ANALYTICS_INSERT
} from "./analytics-schema";
import {
NOOK_DOWNLOAD_PATH,
getWeeklyReport,
pickMover,
readCampaigns,
readFunnel,
readTraffic,
renderWeeklyMarkdown,
windowsFor,
type Connections,
type FunnelCounts,
type MetricsConn
} from "./weekly-metrics";
// Same reasoning as the backfill suite: the queries run real SQL, so an
// in-memory SQLite shaped like production is the only honest stand-in. A stub
// that returned canned rows would pass with a column or a label the live table
// does not have.
const NOOK_DDL = [
`CREATE TABLE trials (
fingerprint TEXT PRIMARY KEY,
started_at TEXT NOT NULL,
email TEXT
)`,
`CREATE TABLE licenses (
id TEXT PRIMARY KEY,
key TEXT UNIQUE NOT NULL,
email TEXT NOT NULL,
stripe_session_id TEXT UNIQUE NOT NULL,
created_at TEXT NOT NULL,
revoked INTEGER NOT NULL DEFAULT 0
)`,
`CREATE TABLE activations (
id TEXT PRIMARY KEY,
license_id TEXT NOT NULL,
device_fingerprint TEXT NOT NULL,
device_name TEXT NOT NULL,
activated_at TEXT NOT NULL,
deactivated_at TEXT
)`
];
function adapter(db: Database): MetricsConn {
return {
async execute(query) {
const args = (query.args ?? []) as never[];
const statement = db.query(query.sql);
if (/^\s*(select|pragma|with)\b/i.test(query.sql)) {
return { rows: statement.all(...args) as unknown[], rowsAffected: 0 };
}
return {
rows: [] as unknown[],
rowsAffected: statement.run(...args).changes
};
}
};
}
let analyticsDb: Database;
let nookDb: Database;
let conns: Connections;
const NOW = new Date("2026-10-29T16:00:00Z");
const row = (
overrides: Record<string, unknown>
): Record<string, unknown> => ({
id: crypto.randomUUID(),
user_id: null,
path: "/",
method: "GET",
referrer: null,
user_agent: "Mozilla/5.0 (Macintosh)",
ip_address: "203.0.113.9",
country: "GB",
device_type: "desktop",
browser: "safari",
os: "macos",
utm_source: null,
utm_medium: null,
utm_campaign: null,
duration_ms: null,
fcp: null,
lcp: null,
cls: null,
fid: null,
inp: null,
ttfb: null,
dom_load: null,
load_complete: null,
...overrides
});
function insertAnalytics(overrides: Record<string, unknown>): void {
const values = row(overrides);
analyticsDb
.prepare(VISITOR_ANALYTICS_INSERT)
.run(...(VISITOR_ANALYTICS_COLUMNS.map((column) => values[column]) as never[]));
// `created_at` is not in VISITOR_ANALYTICS_COLUMNS (it defaults to now), so
// the window each row belongs to has to be set after the insert.
if (values.created_at) {
analyticsDb
.prepare(`UPDATE VisitorAnalytics SET created_at = ? WHERE id = ?`)
.run(values.created_at as string, values.id as string);
}
}
// Inside the current window for NOW - 7 days: 2026-10-22 16:00 → 2026-10-29 16:00.
const THIS_WEEK = "2026-10-27 10:00:00";
// Inside the previous window: 2026-10-15 16:00 → 2026-10-22 16:00.
const LAST_WEEK = "2026-10-18 10:00:00";
beforeEach(() => {
analyticsDb = new Database(":memory:");
analyticsDb.run(VISITOR_ANALYTICS_DDL);
nookDb = new Database(":memory:");
for (const ddl of NOOK_DDL) nookDb.run(ddl);
conns = { analytics: adapter(analyticsDb), nook: adapter(nookDb) };
});
describe("windowsFor", () => {
it("puts the previous window directly before the current one", () => {
const windows = windowsFor(NOW, 7);
expect(windows.current).toEqual({
start: "2026-10-22 16:00:00",
end: "2026-10-29 16:00:00"
});
expect(windows.previous).toEqual({
start: "2026-10-15 16:00:00",
end: "2026-10-22 16:00:00"
});
});
});
describe("readFunnel", () => {
it("counts each stage inside its window and nothing outside it", async () => {
// Page views count; API calls do not.
insertAnalytics({ path: "/", created_at: THIS_WEEK });
insertAnalytics({ path: "/downloads", created_at: THIS_WEEK });
insertAnalytics({ path: "/api/trpc/anything", created_at: THIS_WEEK });
insertAnalytics({ path: "/", created_at: LAST_WEEK });
// The merged download label, click and served alike.
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, created_at: THIS_WEEK });
insertAnalytics({ path: NOOK_DOWNLOAD_PATH, created_at: THIS_WEEK });
// Checkout page doors, including the legacy /nook/checkout path.
insertAnalytics({ path: "/checkout", created_at: THIS_WEEK });
insertAnalytics({ path: "/nook/checkout", created_at: THIS_WEEK });
// The licence DB stores ISO-8601 with milliseconds.
nookDb
.prepare(`INSERT INTO trials (fingerprint, started_at) VALUES (?, ?)`)
.run("fp-1", "2026-10-28T20:28:46.806Z");
nookDb
.prepare(`INSERT INTO trials (fingerprint, started_at) VALUES (?, ?)`)
.run("fp-old", "2026-08-26T19:26:12.791Z");
nookDb
.prepare(
`INSERT INTO licenses (id, key, email, stripe_session_id, created_at)
VALUES (?, ?, ?, ?, ?)`
)
.run("lic-1", "NOOK-1", "buyer@example.com", "cs_1", "2026-10-25T12:00:00.000Z");
nookDb
.prepare(
`INSERT INTO activations (id, license_id, device_fingerprint, device_name, activated_at)
VALUES (?, ?, ?, ?, ?)`
)
.run("act-1", "lic-1", "device-1", "Hermes", "2026-10-25T12:05:00.000Z");
const window = windowsFor(NOW, 7).current;
const funnel = await readFunnel(conns, window);
// "/checkout" and "/nook/checkout" are page views too, so they land in
// both the visit count and the checkout count.
expect(funnel).toEqual({
visit: 4,
download: 2,
trial: 1,
checkout: 2,
purchase: 1,
activation: 1
});
});
it("reports zeros against an empty database", async () => {
const window = windowsFor(NOW, 7).current;
expect(await readFunnel(conns, window)).toEqual({
visit: 0,
download: 0,
trial: 0,
checkout: 0,
purchase: 0,
activation: 0
});
});
});
describe("readTraffic", () => {
it("separates loopback and automation from browser traffic", async () => {
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
ip_address: "127.0.0.1",
user_agent: "curl/8.7.1",
created_at: THIS_WEEK
});
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
ip_address: "::1",
user_agent: "The Nook/0.13.4 Sparkle/2.9.6",
created_at: THIS_WEEK
});
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
ip_address: "203.0.113.9",
user_agent: "Mozilla/5.0 (Macintosh) Safari/605.1.15",
created_at: THIS_WEEK
});
insertAnalytics({ path: "/", ip_address: "203.0.113.9", created_at: THIS_WEEK });
const traffic = await readTraffic(conns, windowsFor(NOW, 7).current);
expect(traffic).toEqual({
analyticsRows: 4,
loopbackRows: 2,
allTimeRows: 4,
allTimeAddressedRows: 4,
allTimeLoopbackRows: 2,
downloadRows: 3,
downloadLoopbackRows: 2,
downloadNonBrowserRows: 2
});
});
});
describe("readCampaigns", () => {
it("groups tagged rows, excludes untagged, and orders by volume", async () => {
insertAnalytics({
path: "/",
utm_source: "hn",
utm_medium: "post",
utm_campaign: "launch",
created_at: THIS_WEEK
});
insertAnalytics({
path: "/",
utm_source: "hn",
utm_medium: "post",
utm_campaign: "launch",
created_at: THIS_WEEK
});
insertAnalytics({
path: "/",
utm_source: "r_macapps",
utm_campaign: "week2",
created_at: THIS_WEEK
});
insertAnalytics({ path: "/", created_at: THIS_WEEK });
const campaigns = await readCampaigns(conns.analytics, windowsFor(NOW, 7).current);
expect(campaigns).toEqual([
{ source: "hn", medium: "post", campaign: "launch", count: 2 },
{ source: "r_macapps", medium: null, campaign: "week2", count: 1 }
]);
});
it("caps the list at ten rows", async () => {
for (let i = 0; i < 12; i += 1) {
insertAnalytics({
path: "/",
utm_source: `source-${i}`,
created_at: THIS_WEEK
});
}
const campaigns = await readCampaigns(conns.analytics, windowsFor(NOW, 7).current);
expect(campaigns.length).toBe(10);
});
});
describe("pickMover", () => {
const base: FunnelCounts = {
visit: 0,
download: 0,
trial: 0,
checkout: 0,
purchase: 0,
activation: 0
};
it("picks the stage with the largest absolute change", () => {
const mover = pickMover(
{ ...base, visit: 61, download: 156, trial: 2 },
{ ...base, visit: 56 }
);
expect(mover).toEqual({
key: "download",
label: "download click → DMG served (one label)",
now: 156,
before: 0,
delta: 156
});
});
it("breaks ties toward the earlier stage", () => {
const mover = pickMover(
{ ...base, visit: 5, download: 5 },
{ ...base, visit: 0, download: 0 }
);
expect(mover?.key).toBe("visit");
});
it("returns null when nothing moved", () => {
expect(pickMover({ ...base, visit: 3 }, { ...base, visit: 3 })).toBeNull();
});
});
describe("getWeeklyReport / renderWeeklyMarkdown", () => {
it("reports a week in the agreed stage order with the computed caveats", async () => {
insertAnalytics({ path: "/", created_at: THIS_WEEK });
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
ip_address: "127.0.0.1",
user_agent: "curl/8.7.1",
created_at: THIS_WEEK
});
insertAnalytics({
path: NOOK_DOWNLOAD_PATH,
ip_address: "127.0.0.1",
user_agent: "curl/8.7.1",
created_at: THIS_WEEK
});
nookDb
.prepare(`INSERT INTO trials (fingerprint, started_at) VALUES (?, ?)`)
.run("fp-1", "2026-10-28T20:28:46.806Z");
const report = await getWeeklyReport(conns, { now: NOW, days: 7 });
const markdown = renderWeeklyMarkdown(report);
expect(report.mover?.key).toBe("download");
expect(markdown).toContain(
"Window: **2026-10-22 16:00:00 → 2026-10-29 16:00:00 UTC** (7 days)"
);
expect(markdown).toContain("| visit (page views) | 1 | 0 | +1 |");
expect(markdown.indexOf("visit (page views)")).toBeLessThan(
markdown.indexOf("download click → DMG served")
);
expect(markdown.indexOf("download click → DMG served")).toBeLessThan(
markdown.indexOf("trial started")
);
expect(markdown).toContain("**download click → DMG served (one label): 0 → 2 (+2).**");
expect(markdown).toContain("2 of 3 rows in the window came from a loopback address");
expect(markdown).toContain("Checkout start is not recorded");
expect(markdown).toContain("There is no host dimension");
});
it("says nothing moved rather than inventing a winner", async () => {
const report = await getWeeklyReport(conns, { now: NOW, days: 7 });
const markdown = renderWeeklyMarkdown(report);
expect(report.mover).toBeNull();
expect(markdown).toContain("_Nothing moved: every stage held its previous-week value._");
expect(markdown).toContain("_No tagged traffic in the window._");
});
it("rejects a window that is not a positive number of days", async () => {
await expect(getWeeklyReport(conns, { now: NOW, days: 0 })).rejects.toThrow(
"days must be a positive number"
);
});
});