downloads: track every door and surface them on /analytics
The analytics page could not answer "how many downloads, of which
product". Three of the four download doors recorded nothing: the tRPC
click proxy only left the generic request row, whose path is the raw URL
with the asset buried in `?batch=1&input={"asset_name":...}`, and the S3
file route that serves Sparkle updates and the Nook app logged nothing
at all. The one door that did log (the Nook redirect) kept a private copy
of the logVisit/enrich/flush sequence, under its own path.
All doors now go through trackDownload, which writes one row per event
under /api/downloads/<asset> and flushes it inside the request — a click
is low-volume and serverless is not guaranteed to run the 30s batch
timer after the response. The file route maps its prefix allowlist to
asset keys, so a version bump stays one asset rather than one per
release. getAnalyticsSummary groups those rows into a downloads block and
the freno.me panel renders it under the overview cards.
History is recovered to the degree it exists: the Nook door's rows and
the tRPC clicks still carry enough to derive from (asset name in the URL).
scripts/backfill-downloads.ts inserts those derived rows with a
`backfill-<source id>` primary key and INSERT OR IGNORE, leaving the
originals untouched, so re-running is a no-op. Run against the live table
it added 66 rows: gaze 42, thenook 19, inputhalo 4, lineage 1.
Tests: new backfill suite (derivation, visitor detail carried over,
idempotency, empty history) and summariseDownloads cases; the Nook route
test now pins the canonical path, and the NOOK-337 allowlist audit reads
the map keys instead of quoted strings. Red-checked by stubbing the
downloads block back to zeros — the summary case fails, then passes.
Full suite matches master.
Co-Authored-By: nw-flash <noreply@nw-flash.local>
MISC-48
This commit is contained in:
@@ -12,6 +12,35 @@
|
||||
* Tests inject a fake `api` and a capture sink.
|
||||
*/
|
||||
|
||||
/** Canonical analytics path for one download event. One label per asset, so a
|
||||
* click on a landing page and a Sparkle update of the same app land in the
|
||||
* same bucket on the analytics page. */
|
||||
export const DOWNLOAD_PATH_PREFIX = "/api/downloads/";
|
||||
|
||||
export function downloadPath(asset: string): string {
|
||||
return `${DOWNLOAD_PATH_PREFIX}${asset}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Group download rows by the asset named in their canonical path, and total
|
||||
* them. Split out from the query so the labelling has a unit test without a
|
||||
* database — the writer and the reader share `DOWNLOAD_PATH_PREFIX`, so the
|
||||
* two cannot drift apart.
|
||||
*/
|
||||
export function summariseDownloads(
|
||||
rows: Array<{ path: string; count: number }>
|
||||
): { total: number; byAsset: Array<{ asset: string; count: number }> } {
|
||||
const byAsset = rows.map((row) => ({
|
||||
asset: row.path.slice(DOWNLOAD_PATH_PREFIX.length),
|
||||
count: row.count
|
||||
}));
|
||||
|
||||
return {
|
||||
total: byAsset.reduce((sum, row) => sum + row.count, 0),
|
||||
byAsset
|
||||
};
|
||||
}
|
||||
|
||||
/** Structural shape of the tRPC downloads proxy this helper depends on. */
|
||||
export interface DownloadApi {
|
||||
downloads: {
|
||||
|
||||
@@ -202,6 +202,32 @@ function FrenomePanel(props: FrenomePanelProps) {
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Downloads — every door logs one row at /api/downloads/<asset> */}
|
||||
<Show when={data.downloads.total > 0}>
|
||||
<div class="bg-surface0 border-surface1 mb-8 rounded-lg border shadow">
|
||||
<div class="border-surface1 border-b p-6">
|
||||
<h2 class="text-text text-2xl font-bold">Downloads</h2>
|
||||
<p class="text-subtext0 mt-1 text-sm">
|
||||
{formatNumber(data.downloads.total)} events in the last
|
||||
{props.days === 1 ? " 24 hours" : ` ${props.days} days`} —
|
||||
landing page clicks and files served, by product
|
||||
</p>
|
||||
</div>
|
||||
<div class="space-y-3 p-6">
|
||||
<For each={data.downloads.byAsset}>
|
||||
{(download) => (
|
||||
<StatBar
|
||||
label={download.asset}
|
||||
count={download.count}
|
||||
total={data.downloads.total}
|
||||
color="bg-teal-600"
|
||||
/>
|
||||
)}
|
||||
</For>
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
{/* Performance */}
|
||||
<Show when={performanceStats()} keyed>
|
||||
{(ps) => (
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { APIEvent } from "@solidjs/start/server";
|
||||
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { env } from "~/env/server";
|
||||
import { trackDownload } from "~/server/track-download";
|
||||
|
||||
/**
|
||||
* Serves macOS app DMG files and delta updates from S3
|
||||
@@ -27,10 +28,20 @@ export async function GET(event: APIEvent) {
|
||||
});
|
||||
}
|
||||
|
||||
const validPrefixes = ["Gaze", "InputHalo", "TheNook", "Nook"];
|
||||
const isValidPrefix = validPrefixes.some((prefix) => filename.startsWith(prefix));
|
||||
// Prefix → canonical asset key, so a version bump stays one asset in
|
||||
// analytics rather than one per release.
|
||||
const assetKeys: Record<string, string> = {
|
||||
Gaze: "gaze",
|
||||
InputHalo: "inputhalo",
|
||||
TheNook: "thenook",
|
||||
Nook: "thenook"
|
||||
};
|
||||
|
||||
const asset = Object.keys(assetKeys).find((prefix) =>
|
||||
filename.startsWith(prefix)
|
||||
);
|
||||
if (
|
||||
!isValidPrefix ||
|
||||
!asset ||
|
||||
(!filename.endsWith(".dmg") &&
|
||||
!filename.endsWith(".delta") &&
|
||||
!filename.endsWith(".zip"))
|
||||
@@ -84,13 +95,20 @@ export async function GET(event: APIEvent) {
|
||||
|
||||
console.log(`✓ Serving ${filename} (${body.length} bytes)`);
|
||||
|
||||
return new Response(body, {
|
||||
await trackDownload(event, assetKeys[asset]!);
|
||||
|
||||
// SAFETY: `transformToByteArray` yields `Uint8Array<ArrayBufferLike>`, which the
|
||||
// DOM lib's `BufferSource` (ArrayBuffer-backed) does not accept, though undici
|
||||
// takes it as-is at runtime. State the intent at the one boundary it hits.
|
||||
return new Response(body as unknown as BodyInit, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": contentType,
|
||||
"Content-Length": body.length.toString(),
|
||||
"Content-Disposition": `attachment; filename="${filename}"`,
|
||||
"Cache-Control": "public, max-age=86400", // Cache for 24 hours
|
||||
// Deliberately open: these are public installer files with no cookies or
|
||||
// credentials, fetched by Sparkle and by direct browser navigation.
|
||||
"Access-Control-Allow-Origin": "*"
|
||||
}
|
||||
});
|
||||
|
||||
@@ -19,14 +19,24 @@ const SOURCE = readFileSync(
|
||||
"utf8"
|
||||
);
|
||||
|
||||
/**
|
||||
* The allowlist moved from an array to a `prefix → asset key` map when download
|
||||
* tracking was added, so read the keys out of the map rather than matching
|
||||
* quoted strings.
|
||||
*/
|
||||
const ALLOWLIST_KEYS = [
|
||||
...(/const assetKeys: Record<string, string> = \{([^}]*)\}/.exec(SOURCE)?.[1] ?? "")
|
||||
.matchAll(/"?([A-Za-z]+)"?\s*:/g)
|
||||
].map((match) => match[1]);
|
||||
|
||||
describe("Downloads proxy filename allowlist (NOOK-337)", () => {
|
||||
it("allows the Nook app-bundle prefix used by Sparkle delta names", () => {
|
||||
expect(SOURCE).toContain('"Nook"');
|
||||
expect(ALLOWLIST_KEYS).toContain("Nook");
|
||||
});
|
||||
|
||||
it("still allows every previously served prefix", () => {
|
||||
for (const prefix of ["Gaze", "InputHalo", "TheNook"]) {
|
||||
expect(SOURCE).toContain(`"${prefix}"`);
|
||||
expect(ALLOWLIST_KEYS).toContain(prefix);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -111,7 +111,7 @@ describe("GET /api/the-nook/download", () => {
|
||||
|
||||
expect(logged.length).toBe(1);
|
||||
expect(logged[0]).toMatchObject({
|
||||
path: "/api/the-nook/download",
|
||||
path: "/api/downloads/thenook",
|
||||
method: "GET",
|
||||
referrer: "https://nook.freno.me/"
|
||||
});
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import type { APIEvent } from "@solidjs/start/server";
|
||||
import { getRequestIP } from "vinxi/http";
|
||||
import { logVisit, enrichAnalyticsEntry, flushAnalytics } from "~/server/analytics";
|
||||
import { parseUtm } from "~/server/utm";
|
||||
import { trackDownload } from "~/server/track-download";
|
||||
import { resolveLatestDownload } from "./_download";
|
||||
|
||||
/**
|
||||
@@ -15,13 +13,11 @@ import { resolveLatestDownload } from "./_download";
|
||||
* Why a redirect and not a direct S3 link: the appcast route and the DMG
|
||||
* route never logged anything, so step 2 of the funnel (download click →
|
||||
* DMG served) was invisible. Every click now lands one row in
|
||||
* VisitorAnalytics under the path below, which is what `topApiCalls`
|
||||
* already surfaces.
|
||||
* VisitorAnalytics under `/api/downloads/thenook`, alongside the clicks and
|
||||
* served files of every other product — see `~/server/track-download`.
|
||||
*
|
||||
* Country is read from the edge headers Vercel/Cloudflare set; the
|
||||
* analytics buffer carries a `country` column that nothing populated
|
||||
* until now. `?c=` is accepted as an override for previews and manual
|
||||
* verification.
|
||||
* Referrer, campaign tags, edge country (with `?c=` override) and user agent
|
||||
* are read by `trackDownload`, which is shared with the other download doors.
|
||||
*
|
||||
* ONLY the handler is exported here: Vinxi's route tree-shake deletes
|
||||
* exported declarations that are not in the router's `pick=` list, so a
|
||||
@@ -29,47 +25,10 @@ import { resolveLatestDownload } from "./_download";
|
||||
* referenced it. All resolution helpers live in `./_download.ts`.
|
||||
*/
|
||||
|
||||
function firstHeader(request: Request, ...names: string[]): string | null {
|
||||
for (const name of names) {
|
||||
const value = request.headers.get(name);
|
||||
if (value) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function GET(event: APIEvent): Promise<Response> {
|
||||
const { request } = event;
|
||||
const url = new URL(request.url);
|
||||
|
||||
const forwarded = firstHeader(request, "x-forwarded-for");
|
||||
const ipAddress =
|
||||
getRequestIP(event.nativeEvent) ??
|
||||
(forwarded ? forwarded.split(",")[0]!.trim() : null);
|
||||
|
||||
// Best-effort: a failed analytics write must never block a download.
|
||||
// Flushed (not merely buffered) because a download click is low-volume and
|
||||
// the batch timer is not guaranteed to survive the response on serverless.
|
||||
try {
|
||||
const referrer = firstHeader(request, "referer", "referrer");
|
||||
await logVisit(
|
||||
enrichAnalyticsEntry({
|
||||
path: "/api/the-nook/download",
|
||||
method: "GET",
|
||||
referrer,
|
||||
userAgent: firstHeader(request, "user-agent"),
|
||||
ipAddress,
|
||||
// The click carries its own tags when a campaign links straight here;
|
||||
// otherwise the landing page's tags ride along in the referer.
|
||||
...parseUtm(request.url, referrer),
|
||||
country:
|
||||
url.searchParams.get("c") ??
|
||||
firstHeader(request, "x-vercel-ip-country", "cf-ipcountry")
|
||||
})
|
||||
);
|
||||
await flushAnalytics();
|
||||
} catch (error) {
|
||||
console.error("Failed to log download click:", error);
|
||||
}
|
||||
// Recorded before resolving: a click that dies at resolution is exactly the
|
||||
// failure the funnel needs to show.
|
||||
await trackDownload(event, "thenook");
|
||||
|
||||
let target: string | null = null;
|
||||
try {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { ConnectionFactory } from "./database";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import type { VisitorAnalytics, AnalyticsQuery } from "~/db/types";
|
||||
import { DOWNLOAD_PATH_PREFIX, summariseDownloads } from "~/lib/download-asset";
|
||||
import { CACHE_CONFIG } from "~/config";
|
||||
import {
|
||||
VISITOR_ANALYTICS_ADDED_COLUMNS,
|
||||
@@ -272,6 +273,10 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
|
||||
topReferrers: Array<{ referrer: string; count: number }>;
|
||||
deviceTypes: Array<{ type: string; count: number }>;
|
||||
browsers: Array<{ browser: string; count: number }>;
|
||||
downloads: {
|
||||
total: number;
|
||||
byAsset: Array<{ asset: string; count: number }>;
|
||||
};
|
||||
topCampaigns: Array<{
|
||||
source: string | null;
|
||||
medium: string | null;
|
||||
@@ -409,6 +414,24 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
|
||||
count: row.count as number
|
||||
}));
|
||||
|
||||
// Downloads: every door records one row under `/api/downloads/<asset>` via
|
||||
// `trackDownload`, so the asset is the path suffix and nothing has to be
|
||||
// parsed out of a query string.
|
||||
const downloadsResult = await conn.execute({
|
||||
sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics
|
||||
WHERE path LIKE '${DOWNLOAD_PATH_PREFIX}%'
|
||||
AND created_at >= datetime('now', '-${days} days')
|
||||
GROUP BY path
|
||||
ORDER BY count DESC`,
|
||||
args: []
|
||||
});
|
||||
const downloads = summariseDownloads(
|
||||
downloadsResult.rows.map((row) => ({
|
||||
path: String(row.path),
|
||||
count: row.count as number
|
||||
}))
|
||||
);
|
||||
|
||||
return {
|
||||
totalVisits,
|
||||
totalPageVisits,
|
||||
@@ -420,7 +443,8 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
|
||||
topReferrers,
|
||||
deviceTypes,
|
||||
browsers,
|
||||
topCampaigns
|
||||
topCampaigns,
|
||||
downloads
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
import { env } from "~/env/server";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { trackDownload } from "~/server/track-download";
|
||||
|
||||
const assets: Record<string, string> = {
|
||||
lineage: "Life and Lineage.apk",
|
||||
@@ -73,7 +74,14 @@ const latestAssets: Record<
|
||||
export const downloadsRouter = createTRPCRouter({
|
||||
getDownloadUrl: publicProcedure
|
||||
.input(z.object({ asset_name: z.string() }))
|
||||
.query(async ({ input }) => {
|
||||
.query(async ({ input, ctx }) => {
|
||||
// Logged before signing: a click on a door that then fails to sign is
|
||||
// exactly the event the funnel needs to show. Unknown asset names are
|
||||
// skipped rather than recorded as a download that never happened.
|
||||
if (assets[input.asset_name] || latestAssets[input.asset_name]) {
|
||||
await trackDownload(ctx.event, input.asset_name);
|
||||
}
|
||||
|
||||
const bucket = env.VITE_DOWNLOAD_BUCKET_STRING;
|
||||
|
||||
const credentials = {
|
||||
|
||||
186
src/server/backfill-downloads.test.ts
Normal file
186
src/server/backfill-downloads.test.ts
Normal file
@@ -0,0 +1,186 @@
|
||||
import { beforeEach, describe, expect, it } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import {
|
||||
VISITOR_ANALYTICS_COLUMNS,
|
||||
VISITOR_ANALYTICS_DDL,
|
||||
VISITOR_ANALYTICS_INDEXES,
|
||||
VISITOR_ANALYTICS_INSERT
|
||||
} from "./analytics-schema";
|
||||
import { summariseDownloads } from "~/lib/download-asset";
|
||||
import { assetFromLegacyPath, backfillDownloads } from "./backfill-downloads";
|
||||
|
||||
// Why a real in-memory SQLite database rather than a stub: the backfill runs
|
||||
// real SQL against the live table shape, and a stub would happily accept a
|
||||
// column list the real table does not have. Same reasoning as
|
||||
// analytics-schema.test.ts.
|
||||
//
|
||||
// Nothing is module-mocked here on purpose: bun's `mock.module` is process-wide,
|
||||
// so stubbing the connection factory would leak into the rate-limit and CSRF
|
||||
// suites that share the same database module.
|
||||
|
||||
let db: Database;
|
||||
|
||||
/** Adapt bun:sqlite to the libsql `execute(stmt | sql)` contract. */
|
||||
const conn = {
|
||||
async execute(query: string | { sql: string; args?: unknown[] }) {
|
||||
const sql = typeof query === "string" ? query : query.sql;
|
||||
const args = (
|
||||
typeof query === "string" ? [] : (query.args ?? [])
|
||||
) as never[];
|
||||
const statement = db.query(sql);
|
||||
if (/^\s*(select|pragma|with)\b/i.test(sql)) {
|
||||
return { rows: statement.all(...args) as unknown[], rowsAffected: 0 };
|
||||
}
|
||||
return {
|
||||
rows: [] as unknown[],
|
||||
rowsAffected: statement.run(...args).changes
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const LEGACY_NOOK = "/api/the-nook/download";
|
||||
|
||||
const trpcClick = (asset: string) =>
|
||||
`/api/trpc/downloads.getDownloadUrl?batch=1&input=%7B%220%22%3A%7B%22asset_name%22%3A%22${asset}%22%7D%7D`;
|
||||
|
||||
/** Columns whose value varies per row; everything else stays NULL. */
|
||||
const fixedColumns: Record<string, string> = {
|
||||
method: "GET",
|
||||
referrer: "https://gaze.freno.me/",
|
||||
user_agent: "Mozilla/5.0 (Macintosh)",
|
||||
ip_address: "203.0.113.7",
|
||||
country: "GB"
|
||||
};
|
||||
|
||||
function insertLegacy(id: string, path: string) {
|
||||
const values = VISITOR_ANALYTICS_COLUMNS.map((column) => {
|
||||
if (column === "id") return id;
|
||||
if (column === "path") return path;
|
||||
return fixedColumns[column] ?? null;
|
||||
});
|
||||
db.query(VISITOR_ANALYTICS_INSERT).run(...(values as never[]));
|
||||
}
|
||||
|
||||
function canonicalPaths(): string[] {
|
||||
return db
|
||||
.query<{ path: string }, []>(
|
||||
`SELECT path FROM VisitorAnalytics WHERE path LIKE '/api/downloads/%' ORDER BY path`
|
||||
)
|
||||
.all()
|
||||
.map((r: { path: string }) => r.path);
|
||||
}
|
||||
|
||||
function legacyRowCount(): number {
|
||||
return db
|
||||
.query<
|
||||
{ n: number },
|
||||
[string]
|
||||
>(`SELECT COUNT(*) AS n FROM VisitorAnalytics WHERE path = ?`)
|
||||
.get(LEGACY_NOOK)!.n;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
db = new Database(":memory:");
|
||||
db.exec(VISITOR_ANALYTICS_DDL);
|
||||
for (const index of VISITOR_ANALYTICS_INDEXES) db.exec(index);
|
||||
});
|
||||
|
||||
describe("backfillDownloads", () => {
|
||||
it("derives a canonical row from each recoverable kind of history", async () => {
|
||||
insertLegacy("nook-1", LEGACY_NOOK);
|
||||
insertLegacy("gaze-1", trpcClick("gaze"));
|
||||
insertLegacy("halo-1", trpcClick("inputhalo"));
|
||||
insertLegacy("page-1", "/nook/");
|
||||
|
||||
const added = Object.fromEntries(
|
||||
(await backfillDownloads(conn)).map((r) => [r.asset, r.added])
|
||||
);
|
||||
|
||||
expect(added).toEqual({ thenook: 1, gaze: 1, inputhalo: 1 });
|
||||
expect(canonicalPaths()).toEqual([
|
||||
"/api/downloads/gaze",
|
||||
"/api/downloads/inputhalo",
|
||||
"/api/downloads/thenook"
|
||||
]);
|
||||
// A page visit is not download history and gains no derived row.
|
||||
expect(
|
||||
db
|
||||
.query(`SELECT * FROM VisitorAnalytics WHERE id = 'backfill-page-1'`)
|
||||
.all()
|
||||
).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("keeps the visitor detail of the row it derives from", async () => {
|
||||
insertLegacy("nook-1", LEGACY_NOOK);
|
||||
|
||||
await backfillDownloads(conn);
|
||||
|
||||
expect(
|
||||
db
|
||||
.query<
|
||||
{ country: string; referrer: string },
|
||||
[]
|
||||
>(`SELECT country, referrer FROM VisitorAnalytics WHERE id = 'backfill-nook-1'`)
|
||||
.get()
|
||||
).toEqual({ country: "GB", referrer: "https://gaze.freno.me/" });
|
||||
});
|
||||
|
||||
it("is idempotent and never rewrites the source rows", async () => {
|
||||
insertLegacy("nook-1", LEGACY_NOOK);
|
||||
|
||||
await backfillDownloads(conn);
|
||||
const second = await backfillDownloads(conn);
|
||||
|
||||
expect(second).toEqual([]);
|
||||
expect(canonicalPaths()).toHaveLength(1);
|
||||
expect(legacyRowCount()).toBe(1);
|
||||
});
|
||||
|
||||
it("derives nothing when there is no legacy history", async () => {
|
||||
expect(await backfillDownloads(conn)).toEqual([]);
|
||||
expect(canonicalPaths()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("assetFromLegacyPath", () => {
|
||||
it("maps the Nook door to its canonical asset", () => {
|
||||
expect(assetFromLegacyPath(LEGACY_NOOK)).toBe("thenook");
|
||||
});
|
||||
|
||||
it("reads the asset out of an encoded or a plain tRPC request URL", () => {
|
||||
expect(assetFromLegacyPath(trpcClick("gaze"))).toBe("gaze");
|
||||
expect(
|
||||
assetFromLegacyPath(
|
||||
'/api/trpc/downloads.getDownloadUrl?batch=1&input={"0":{"asset_name":"lineage"}}'
|
||||
)
|
||||
).toBe("lineage");
|
||||
});
|
||||
|
||||
it("returns null when there is nothing to derive", () => {
|
||||
expect(assetFromLegacyPath("/nook/")).toBeNull();
|
||||
expect(
|
||||
assetFromLegacyPath("/api/trpc/downloads.getDownloadUrl")
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("summariseDownloads", () => {
|
||||
it("labels each row by the asset in its path and totals them", () => {
|
||||
expect(
|
||||
summariseDownloads([
|
||||
{ path: "/api/downloads/gaze", count: 4 },
|
||||
{ path: "/api/downloads/thenook", count: 1 }
|
||||
])
|
||||
).toEqual({
|
||||
total: 5,
|
||||
byAsset: [
|
||||
{ asset: "gaze", count: 4 },
|
||||
{ asset: "thenook", count: 1 }
|
||||
]
|
||||
});
|
||||
});
|
||||
|
||||
it("reports nothing rather than throwing on an empty table", () => {
|
||||
expect(summariseDownloads([])).toEqual({ total: 0, byAsset: [] });
|
||||
});
|
||||
});
|
||||
95
src/server/backfill-downloads.ts
Normal file
95
src/server/backfill-downloads.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import { VISITOR_ANALYTICS_COLUMNS } from "~/server/analytics-schema";
|
||||
import { downloadPath } from "~/lib/download-asset";
|
||||
|
||||
/**
|
||||
* Backfill canonical `downloadPath(asset)` rows from download history that was
|
||||
* recorded before `trackDownload` existed.
|
||||
*
|
||||
* Two shapes of history are recoverable, and only these two:
|
||||
*
|
||||
* - the Nook door's own rows (`/api/the-nook/download`), and
|
||||
* - tRPC clicks, where the asset name still sits in the request URL as
|
||||
* `?input={"0":{"asset_name":"gaze"}}` (percent-encoded in practice).
|
||||
*
|
||||
* Every other door wrote nothing, so there is nothing to derive for them.
|
||||
*
|
||||
* Derived rows keep the source row's timestamp, visitor, campaign and country,
|
||||
* get `backfill-<source id>` as primary key and are written with
|
||||
* `INSERT OR IGNORE` — so a second run adds nothing, and the original rows are
|
||||
* left untouched (they are genuine API traffic, and the analytics page only
|
||||
* counts the canonical path).
|
||||
*/
|
||||
|
||||
const LEGACY_NOOK_PATH = "/api/the-nook/download";
|
||||
const TRPC_DOOR_PREFIX = "/api/trpc/downloads.getDownloadUrl";
|
||||
|
||||
/** The two libsql calls used here, narrow enough for an in-memory test client. */
|
||||
export interface DownloadBackfillConn {
|
||||
execute(query: {
|
||||
sql: string;
|
||||
// `any[]` not `unknown[]`: the libsql client types its bind args as InArg[],
|
||||
// and a narrower array type here would reject the real connection.
|
||||
args?: any[];
|
||||
}): Promise<{ rows: unknown[]; rowsAffected?: number }>;
|
||||
}
|
||||
|
||||
export interface DownloadBackfillResult {
|
||||
asset: string;
|
||||
added: number;
|
||||
}
|
||||
|
||||
/** One statement: copy every column of the source row, override id and path. */
|
||||
const INSERT_DERIVED = `INSERT OR IGNORE INTO VisitorAnalytics (${VISITOR_ANALYTICS_COLUMNS.join(", ")})
|
||||
SELECT ${VISITOR_ANALYTICS_COLUMNS.map((column) =>
|
||||
column === "id"
|
||||
? "'backfill-' || id"
|
||||
: column === "path"
|
||||
? "?"
|
||||
: `"${column}"`
|
||||
).join(", ")}
|
||||
FROM VisitorAnalytics
|
||||
WHERE id = ?`;
|
||||
|
||||
/** Pull the asset name back out of a logged tRPC request URL. */
|
||||
export function assetFromLegacyPath(path: string): string | null {
|
||||
if (path === LEGACY_NOOK_PATH) return "thenook";
|
||||
|
||||
let decoded = path;
|
||||
try {
|
||||
decoded = decodeURIComponent(path);
|
||||
} catch {
|
||||
// A malformed escape is not a reason to drop the row — match on the raw.
|
||||
}
|
||||
|
||||
return /"asset_name"\s*:\s*"([^"]+)"/.exec(decoded)?.[1] ?? null;
|
||||
}
|
||||
|
||||
export async function backfillDownloads(
|
||||
conn: DownloadBackfillConn
|
||||
): Promise<DownloadBackfillResult[]> {
|
||||
const legacy = await conn.execute({
|
||||
sql: `SELECT id, path FROM VisitorAnalytics WHERE path = ? OR path LIKE ?`,
|
||||
args: [LEGACY_NOOK_PATH, `${TRPC_DOOR_PREFIX}%`]
|
||||
});
|
||||
|
||||
const added = new Map<string, number>();
|
||||
|
||||
for (const row of legacy.rows) {
|
||||
const { id, path } = row as { id: string; path: string };
|
||||
const asset = assetFromLegacyPath(path);
|
||||
if (!asset) continue;
|
||||
|
||||
const result = await conn.execute({
|
||||
sql: INSERT_DERIVED,
|
||||
args: [downloadPath(asset), id]
|
||||
});
|
||||
|
||||
if (result.rowsAffected) {
|
||||
added.set(asset, (added.get(asset) ?? 0) + result.rowsAffected);
|
||||
}
|
||||
}
|
||||
|
||||
return [...added.entries()]
|
||||
.map(([asset, count]) => ({ asset, added: count }))
|
||||
.sort((a, b) => b.added - a.added);
|
||||
}
|
||||
61
src/server/track-download.ts
Normal file
61
src/server/track-download.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
import type { APIEvent } from "@solidjs/start/server";
|
||||
import { getRequestIP } from "vinxi/http";
|
||||
import {
|
||||
logVisit,
|
||||
enrichAnalyticsEntry,
|
||||
flushAnalytics
|
||||
} from "~/server/analytics";
|
||||
import { parseUtm } from "~/server/utm";
|
||||
import { downloadPath } from "~/lib/download-asset";
|
||||
|
||||
/**
|
||||
* Record one download event under the canonical path `/api/downloads/<asset>`.
|
||||
*
|
||||
* Every download door funnels through here so the analytics page has a single
|
||||
* label per asset to group on: the tRPC click proxy, the S3 file route that
|
||||
* serves Sparkle updates, and the Nook door.
|
||||
*
|
||||
* Best-effort by design — a failed analytics write must never block a
|
||||
* download. Flushed rather than merely buffered: a download is low-volume and
|
||||
* the 30s batch timer is not guaranteed to survive the response on serverless.
|
||||
*
|
||||
* `event.request` is expected to be a Fetch `Request` (true for Vinxi route
|
||||
* handlers and browser tRPC calls). A server-side `createCaller` hands over a
|
||||
* Node request, whose headers have no `get` — that throw is swallowed below,
|
||||
* so such a click is not recorded. No server-side caller downloads today.
|
||||
*/
|
||||
export async function trackDownload(
|
||||
event: APIEvent,
|
||||
asset: string
|
||||
): Promise<void> {
|
||||
const { request } = event;
|
||||
|
||||
try {
|
||||
const referrer =
|
||||
request.headers.get("referer") ?? request.headers.get("referrer");
|
||||
const forwarded = request.headers.get("x-forwarded-for");
|
||||
|
||||
await logVisit(
|
||||
enrichAnalyticsEntry({
|
||||
path: downloadPath(asset),
|
||||
method: "GET",
|
||||
referrer,
|
||||
userAgent: request.headers.get("user-agent"),
|
||||
ipAddress:
|
||||
getRequestIP(event.nativeEvent) ??
|
||||
(forwarded ? forwarded.split(",")[0]!.trim() : null),
|
||||
// The click carries its own tags when a campaign links straight at the
|
||||
// door; otherwise the landing page's tags ride along in the referer.
|
||||
...parseUtm(request.url, referrer),
|
||||
// Edge header country, overridable with ?c= for manual verification.
|
||||
country:
|
||||
new URL(request.url).searchParams.get("c") ??
|
||||
request.headers.get("x-vercel-ip-country") ??
|
||||
request.headers.get("cf-ipcountry")
|
||||
})
|
||||
);
|
||||
await flushAnalytics();
|
||||
} catch (error) {
|
||||
console.error(`Failed to log download of ${asset}:`, error);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user