forked from Mike/freno-dev
downloads: track every door and surface them on /analytics
The analytics page could not answer "how many downloads, of which
product". Three of the four download doors recorded nothing: the tRPC
click proxy only left the generic request row, whose path is the raw URL
with the asset buried in `?batch=1&input={"asset_name":...}`, and the S3
file route that serves Sparkle updates and the Nook app logged nothing
at all. The one door that did log (the Nook redirect) kept a private copy
of the logVisit/enrich/flush sequence, under its own path.
All doors now go through trackDownload, which writes one row per event
under /api/downloads/<asset> and flushes it inside the request — a click
is low-volume and serverless is not guaranteed to run the 30s batch
timer after the response. The file route maps its prefix allowlist to
asset keys, so a version bump stays one asset rather than one per
release. getAnalyticsSummary groups those rows into a downloads block and
the freno.me panel renders it under the overview cards.
History is recovered to the degree it exists: the Nook door's rows and
the tRPC clicks still carry enough to derive from (asset name in the URL).
scripts/backfill-downloads.ts inserts those derived rows with a
`backfill-<source id>` primary key and INSERT OR IGNORE, leaving the
originals untouched, so re-running is a no-op. Run against the live table
it added 66 rows: gaze 42, thenook 19, inputhalo 4, lineage 1.
Tests: new backfill suite (derivation, visitor detail carried over,
idempotency, empty history) and summariseDownloads cases; the Nook route
test now pins the canonical path, and the NOOK-337 allowlist audit reads
the map keys instead of quoted strings. Red-checked by stubbing the
downloads block back to zeros — the summary case fails, then passes.
Full suite matches master.
Co-Authored-By: nw-flash <noreply@nw-flash.local>
MISC-48
This commit is contained in:
21
scripts/backfill-downloads.ts
Normal file
21
scripts/backfill-downloads.ts
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
import { ConnectionFactory } from "~/server/db-connections";
|
||||||
|
import { backfillDownloads } from "~/server/backfill-downloads";
|
||||||
|
|
||||||
|
// One-off: derive canonical /api/downloads/<asset> rows from download history
|
||||||
|
// recorded before `trackDownload` existed. Idempotent — re-running adds
|
||||||
|
// nothing, because derived rows carry a `backfill-<source id>` primary key.
|
||||||
|
//
|
||||||
|
// bun --env-file=.env scripts/backfill-downloads.ts
|
||||||
|
|
||||||
|
const result = await backfillDownloads(ConnectionFactory());
|
||||||
|
|
||||||
|
if (result.length === 0) {
|
||||||
|
console.log("Nothing to backfill: no legacy download rows left to derive.");
|
||||||
|
} else {
|
||||||
|
for (const { asset, added } of result) {
|
||||||
|
console.log(`${asset}: +${added}`);
|
||||||
|
}
|
||||||
|
console.log(
|
||||||
|
`Total: +${result.reduce((sum, row) => sum + row.added, 0)} derived download rows.`
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -12,6 +12,35 @@
|
|||||||
* Tests inject a fake `api` and a capture sink.
|
* Tests inject a fake `api` and a capture sink.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/** Canonical analytics path for one download event. One label per asset, so a
|
||||||
|
* click on a landing page and a Sparkle update of the same app land in the
|
||||||
|
* same bucket on the analytics page. */
|
||||||
|
export const DOWNLOAD_PATH_PREFIX = "/api/downloads/";
|
||||||
|
|
||||||
|
export function downloadPath(asset: string): string {
|
||||||
|
return `${DOWNLOAD_PATH_PREFIX}${asset}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Group download rows by the asset named in their canonical path, and total
|
||||||
|
* them. Split out from the query so the labelling has a unit test without a
|
||||||
|
* database — the writer and the reader share `DOWNLOAD_PATH_PREFIX`, so the
|
||||||
|
* two cannot drift apart.
|
||||||
|
*/
|
||||||
|
export function summariseDownloads(
|
||||||
|
rows: Array<{ path: string; count: number }>
|
||||||
|
): { total: number; byAsset: Array<{ asset: string; count: number }> } {
|
||||||
|
const byAsset = rows.map((row) => ({
|
||||||
|
asset: row.path.slice(DOWNLOAD_PATH_PREFIX.length),
|
||||||
|
count: row.count
|
||||||
|
}));
|
||||||
|
|
||||||
|
return {
|
||||||
|
total: byAsset.reduce((sum, row) => sum + row.count, 0),
|
||||||
|
byAsset
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/** Structural shape of the tRPC downloads proxy this helper depends on. */
|
/** Structural shape of the tRPC downloads proxy this helper depends on. */
|
||||||
export interface DownloadApi {
|
export interface DownloadApi {
|
||||||
downloads: {
|
downloads: {
|
||||||
|
|||||||
@@ -202,6 +202,32 @@ function FrenomePanel(props: FrenomePanelProps) {
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Downloads — every door logs one row at /api/downloads/<asset> */}
|
||||||
|
<Show when={data.downloads.total > 0}>
|
||||||
|
<div class="bg-surface0 border-surface1 mb-8 rounded-lg border shadow">
|
||||||
|
<div class="border-surface1 border-b p-6">
|
||||||
|
<h2 class="text-text text-2xl font-bold">Downloads</h2>
|
||||||
|
<p class="text-subtext0 mt-1 text-sm">
|
||||||
|
{formatNumber(data.downloads.total)} events in the last
|
||||||
|
{props.days === 1 ? " 24 hours" : ` ${props.days} days`} —
|
||||||
|
landing page clicks and files served, by product
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div class="space-y-3 p-6">
|
||||||
|
<For each={data.downloads.byAsset}>
|
||||||
|
{(download) => (
|
||||||
|
<StatBar
|
||||||
|
label={download.asset}
|
||||||
|
count={download.count}
|
||||||
|
total={data.downloads.total}
|
||||||
|
color="bg-teal-600"
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</For>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</Show>
|
||||||
|
|
||||||
{/* Performance */}
|
{/* Performance */}
|
||||||
<Show when={performanceStats()} keyed>
|
<Show when={performanceStats()} keyed>
|
||||||
{(ps) => (
|
{(ps) => (
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import type { APIEvent } from "@solidjs/start/server";
|
import type { APIEvent } from "@solidjs/start/server";
|
||||||
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
|
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
|
||||||
import { env } from "~/env/server";
|
import { env } from "~/env/server";
|
||||||
|
import { trackDownload } from "~/server/track-download";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Serves macOS app DMG files and delta updates from S3
|
* Serves macOS app DMG files and delta updates from S3
|
||||||
@@ -27,10 +28,20 @@ export async function GET(event: APIEvent) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const validPrefixes = ["Gaze", "InputHalo", "TheNook", "Nook"];
|
// Prefix → canonical asset key, so a version bump stays one asset in
|
||||||
const isValidPrefix = validPrefixes.some((prefix) => filename.startsWith(prefix));
|
// analytics rather than one per release.
|
||||||
|
const assetKeys: Record<string, string> = {
|
||||||
|
Gaze: "gaze",
|
||||||
|
InputHalo: "inputhalo",
|
||||||
|
TheNook: "thenook",
|
||||||
|
Nook: "thenook"
|
||||||
|
};
|
||||||
|
|
||||||
|
const asset = Object.keys(assetKeys).find((prefix) =>
|
||||||
|
filename.startsWith(prefix)
|
||||||
|
);
|
||||||
if (
|
if (
|
||||||
!isValidPrefix ||
|
!asset ||
|
||||||
(!filename.endsWith(".dmg") &&
|
(!filename.endsWith(".dmg") &&
|
||||||
!filename.endsWith(".delta") &&
|
!filename.endsWith(".delta") &&
|
||||||
!filename.endsWith(".zip"))
|
!filename.endsWith(".zip"))
|
||||||
@@ -84,13 +95,20 @@ export async function GET(event: APIEvent) {
|
|||||||
|
|
||||||
console.log(`✓ Serving ${filename} (${body.length} bytes)`);
|
console.log(`✓ Serving ${filename} (${body.length} bytes)`);
|
||||||
|
|
||||||
return new Response(body, {
|
await trackDownload(event, assetKeys[asset]!);
|
||||||
|
|
||||||
|
// SAFETY: `transformToByteArray` yields `Uint8Array<ArrayBufferLike>`, which the
|
||||||
|
// DOM lib's `BufferSource` (ArrayBuffer-backed) does not accept, though undici
|
||||||
|
// takes it as-is at runtime. State the intent at the one boundary it hits.
|
||||||
|
return new Response(body as unknown as BodyInit, {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: {
|
headers: {
|
||||||
"Content-Type": contentType,
|
"Content-Type": contentType,
|
||||||
"Content-Length": body.length.toString(),
|
"Content-Length": body.length.toString(),
|
||||||
"Content-Disposition": `attachment; filename="${filename}"`,
|
"Content-Disposition": `attachment; filename="${filename}"`,
|
||||||
"Cache-Control": "public, max-age=86400", // Cache for 24 hours
|
"Cache-Control": "public, max-age=86400", // Cache for 24 hours
|
||||||
|
// Deliberately open: these are public installer files with no cookies or
|
||||||
|
// credentials, fetched by Sparkle and by direct browser navigation.
|
||||||
"Access-Control-Allow-Origin": "*"
|
"Access-Control-Allow-Origin": "*"
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -19,14 +19,24 @@ const SOURCE = readFileSync(
|
|||||||
"utf8"
|
"utf8"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The allowlist moved from an array to a `prefix → asset key` map when download
|
||||||
|
* tracking was added, so read the keys out of the map rather than matching
|
||||||
|
* quoted strings.
|
||||||
|
*/
|
||||||
|
const ALLOWLIST_KEYS = [
|
||||||
|
...(/const assetKeys: Record<string, string> = \{([^}]*)\}/.exec(SOURCE)?.[1] ?? "")
|
||||||
|
.matchAll(/"?([A-Za-z]+)"?\s*:/g)
|
||||||
|
].map((match) => match[1]);
|
||||||
|
|
||||||
describe("Downloads proxy filename allowlist (NOOK-337)", () => {
|
describe("Downloads proxy filename allowlist (NOOK-337)", () => {
|
||||||
it("allows the Nook app-bundle prefix used by Sparkle delta names", () => {
|
it("allows the Nook app-bundle prefix used by Sparkle delta names", () => {
|
||||||
expect(SOURCE).toContain('"Nook"');
|
expect(ALLOWLIST_KEYS).toContain("Nook");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("still allows every previously served prefix", () => {
|
it("still allows every previously served prefix", () => {
|
||||||
for (const prefix of ["Gaze", "InputHalo", "TheNook"]) {
|
for (const prefix of ["Gaze", "InputHalo", "TheNook"]) {
|
||||||
expect(SOURCE).toContain(`"${prefix}"`);
|
expect(ALLOWLIST_KEYS).toContain(prefix);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ describe("GET /api/the-nook/download", () => {
|
|||||||
|
|
||||||
expect(logged.length).toBe(1);
|
expect(logged.length).toBe(1);
|
||||||
expect(logged[0]).toMatchObject({
|
expect(logged[0]).toMatchObject({
|
||||||
path: "/api/the-nook/download",
|
path: "/api/downloads/thenook",
|
||||||
method: "GET",
|
method: "GET",
|
||||||
referrer: "https://nook.freno.me/"
|
referrer: "https://nook.freno.me/"
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
import type { APIEvent } from "@solidjs/start/server";
|
import type { APIEvent } from "@solidjs/start/server";
|
||||||
import { getRequestIP } from "vinxi/http";
|
import { trackDownload } from "~/server/track-download";
|
||||||
import { logVisit, enrichAnalyticsEntry, flushAnalytics } from "~/server/analytics";
|
|
||||||
import { parseUtm } from "~/server/utm";
|
|
||||||
import { resolveLatestDownload } from "./_download";
|
import { resolveLatestDownload } from "./_download";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -15,13 +13,11 @@ import { resolveLatestDownload } from "./_download";
|
|||||||
* Why a redirect and not a direct S3 link: the appcast route and the DMG
|
* Why a redirect and not a direct S3 link: the appcast route and the DMG
|
||||||
* route never logged anything, so step 2 of the funnel (download click →
|
* route never logged anything, so step 2 of the funnel (download click →
|
||||||
* DMG served) was invisible. Every click now lands one row in
|
* DMG served) was invisible. Every click now lands one row in
|
||||||
* VisitorAnalytics under the path below, which is what `topApiCalls`
|
* VisitorAnalytics under `/api/downloads/thenook`, alongside the clicks and
|
||||||
* already surfaces.
|
* served files of every other product — see `~/server/track-download`.
|
||||||
*
|
*
|
||||||
* Country is read from the edge headers Vercel/Cloudflare set; the
|
* Referrer, campaign tags, edge country (with `?c=` override) and user agent
|
||||||
* analytics buffer carries a `country` column that nothing populated
|
* are read by `trackDownload`, which is shared with the other download doors.
|
||||||
* until now. `?c=` is accepted as an override for previews and manual
|
|
||||||
* verification.
|
|
||||||
*
|
*
|
||||||
* ONLY the handler is exported here: Vinxi's route tree-shake deletes
|
* ONLY the handler is exported here: Vinxi's route tree-shake deletes
|
||||||
* exported declarations that are not in the router's `pick=` list, so a
|
* exported declarations that are not in the router's `pick=` list, so a
|
||||||
@@ -29,47 +25,10 @@ import { resolveLatestDownload } from "./_download";
|
|||||||
* referenced it. All resolution helpers live in `./_download.ts`.
|
* referenced it. All resolution helpers live in `./_download.ts`.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
function firstHeader(request: Request, ...names: string[]): string | null {
|
|
||||||
for (const name of names) {
|
|
||||||
const value = request.headers.get(name);
|
|
||||||
if (value) return value;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function GET(event: APIEvent): Promise<Response> {
|
export async function GET(event: APIEvent): Promise<Response> {
|
||||||
const { request } = event;
|
// Recorded before resolving: a click that dies at resolution is exactly the
|
||||||
const url = new URL(request.url);
|
// failure the funnel needs to show.
|
||||||
|
await trackDownload(event, "thenook");
|
||||||
const forwarded = firstHeader(request, "x-forwarded-for");
|
|
||||||
const ipAddress =
|
|
||||||
getRequestIP(event.nativeEvent) ??
|
|
||||||
(forwarded ? forwarded.split(",")[0]!.trim() : null);
|
|
||||||
|
|
||||||
// Best-effort: a failed analytics write must never block a download.
|
|
||||||
// Flushed (not merely buffered) because a download click is low-volume and
|
|
||||||
// the batch timer is not guaranteed to survive the response on serverless.
|
|
||||||
try {
|
|
||||||
const referrer = firstHeader(request, "referer", "referrer");
|
|
||||||
await logVisit(
|
|
||||||
enrichAnalyticsEntry({
|
|
||||||
path: "/api/the-nook/download",
|
|
||||||
method: "GET",
|
|
||||||
referrer,
|
|
||||||
userAgent: firstHeader(request, "user-agent"),
|
|
||||||
ipAddress,
|
|
||||||
// The click carries its own tags when a campaign links straight here;
|
|
||||||
// otherwise the landing page's tags ride along in the referer.
|
|
||||||
...parseUtm(request.url, referrer),
|
|
||||||
country:
|
|
||||||
url.searchParams.get("c") ??
|
|
||||||
firstHeader(request, "x-vercel-ip-country", "cf-ipcountry")
|
|
||||||
})
|
|
||||||
);
|
|
||||||
await flushAnalytics();
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Failed to log download click:", error);
|
|
||||||
}
|
|
||||||
|
|
||||||
let target: string | null = null;
|
let target: string | null = null;
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { ConnectionFactory } from "./database";
|
import { ConnectionFactory } from "./database";
|
||||||
import { v4 as uuid } from "uuid";
|
import { v4 as uuid } from "uuid";
|
||||||
import type { VisitorAnalytics, AnalyticsQuery } from "~/db/types";
|
import type { VisitorAnalytics, AnalyticsQuery } from "~/db/types";
|
||||||
|
import { DOWNLOAD_PATH_PREFIX, summariseDownloads } from "~/lib/download-asset";
|
||||||
import { CACHE_CONFIG } from "~/config";
|
import { CACHE_CONFIG } from "~/config";
|
||||||
import {
|
import {
|
||||||
VISITOR_ANALYTICS_ADDED_COLUMNS,
|
VISITOR_ANALYTICS_ADDED_COLUMNS,
|
||||||
@@ -272,6 +273,10 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
|
|||||||
topReferrers: Array<{ referrer: string; count: number }>;
|
topReferrers: Array<{ referrer: string; count: number }>;
|
||||||
deviceTypes: Array<{ type: string; count: number }>;
|
deviceTypes: Array<{ type: string; count: number }>;
|
||||||
browsers: Array<{ browser: string; count: number }>;
|
browsers: Array<{ browser: string; count: number }>;
|
||||||
|
downloads: {
|
||||||
|
total: number;
|
||||||
|
byAsset: Array<{ asset: string; count: number }>;
|
||||||
|
};
|
||||||
topCampaigns: Array<{
|
topCampaigns: Array<{
|
||||||
source: string | null;
|
source: string | null;
|
||||||
medium: string | null;
|
medium: string | null;
|
||||||
@@ -409,6 +414,24 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
|
|||||||
count: row.count as number
|
count: row.count as number
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
// Downloads: every door records one row under `/api/downloads/<asset>` via
|
||||||
|
// `trackDownload`, so the asset is the path suffix and nothing has to be
|
||||||
|
// parsed out of a query string.
|
||||||
|
const downloadsResult = await conn.execute({
|
||||||
|
sql: `SELECT path, COUNT(*) as count FROM VisitorAnalytics
|
||||||
|
WHERE path LIKE '${DOWNLOAD_PATH_PREFIX}%'
|
||||||
|
AND created_at >= datetime('now', '-${days} days')
|
||||||
|
GROUP BY path
|
||||||
|
ORDER BY count DESC`,
|
||||||
|
args: []
|
||||||
|
});
|
||||||
|
const downloads = summariseDownloads(
|
||||||
|
downloadsResult.rows.map((row) => ({
|
||||||
|
path: String(row.path),
|
||||||
|
count: row.count as number
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
totalVisits,
|
totalVisits,
|
||||||
totalPageVisits,
|
totalPageVisits,
|
||||||
@@ -420,7 +443,8 @@ export async function getAnalyticsSummary(days: number = 30): Promise<{
|
|||||||
topReferrers,
|
topReferrers,
|
||||||
deviceTypes,
|
deviceTypes,
|
||||||
browsers,
|
browsers,
|
||||||
topCampaigns
|
topCampaigns,
|
||||||
|
downloads
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||||
import { env } from "~/env/server";
|
import { env } from "~/env/server";
|
||||||
import { TRPCError } from "@trpc/server";
|
import { TRPCError } from "@trpc/server";
|
||||||
|
import { trackDownload } from "~/server/track-download";
|
||||||
|
|
||||||
const assets: Record<string, string> = {
|
const assets: Record<string, string> = {
|
||||||
lineage: "Life and Lineage.apk",
|
lineage: "Life and Lineage.apk",
|
||||||
@@ -73,7 +74,14 @@ const latestAssets: Record<
|
|||||||
export const downloadsRouter = createTRPCRouter({
|
export const downloadsRouter = createTRPCRouter({
|
||||||
getDownloadUrl: publicProcedure
|
getDownloadUrl: publicProcedure
|
||||||
.input(z.object({ asset_name: z.string() }))
|
.input(z.object({ asset_name: z.string() }))
|
||||||
.query(async ({ input }) => {
|
.query(async ({ input, ctx }) => {
|
||||||
|
// Logged before signing: a click on a door that then fails to sign is
|
||||||
|
// exactly the event the funnel needs to show. Unknown asset names are
|
||||||
|
// skipped rather than recorded as a download that never happened.
|
||||||
|
if (assets[input.asset_name] || latestAssets[input.asset_name]) {
|
||||||
|
await trackDownload(ctx.event, input.asset_name);
|
||||||
|
}
|
||||||
|
|
||||||
const bucket = env.VITE_DOWNLOAD_BUCKET_STRING;
|
const bucket = env.VITE_DOWNLOAD_BUCKET_STRING;
|
||||||
|
|
||||||
const credentials = {
|
const credentials = {
|
||||||
|
|||||||
186
src/server/backfill-downloads.test.ts
Normal file
186
src/server/backfill-downloads.test.ts
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
import { beforeEach, describe, expect, it } from "bun:test";
|
||||||
|
import { Database } from "bun:sqlite";
|
||||||
|
import {
|
||||||
|
VISITOR_ANALYTICS_COLUMNS,
|
||||||
|
VISITOR_ANALYTICS_DDL,
|
||||||
|
VISITOR_ANALYTICS_INDEXES,
|
||||||
|
VISITOR_ANALYTICS_INSERT
|
||||||
|
} from "./analytics-schema";
|
||||||
|
import { summariseDownloads } from "~/lib/download-asset";
|
||||||
|
import { assetFromLegacyPath, backfillDownloads } from "./backfill-downloads";
|
||||||
|
|
||||||
|
// Why a real in-memory SQLite database rather than a stub: the backfill runs
|
||||||
|
// real SQL against the live table shape, and a stub would happily accept a
|
||||||
|
// column list the real table does not have. Same reasoning as
|
||||||
|
// analytics-schema.test.ts.
|
||||||
|
//
|
||||||
|
// Nothing is module-mocked here on purpose: bun's `mock.module` is process-wide,
|
||||||
|
// so stubbing the connection factory would leak into the rate-limit and CSRF
|
||||||
|
// suites that share the same database module.
|
||||||
|
|
||||||
|
let db: Database;
|
||||||
|
|
||||||
|
/** Adapt bun:sqlite to the libsql `execute(stmt | sql)` contract. */
|
||||||
|
const conn = {
|
||||||
|
async execute(query: string | { sql: string; args?: unknown[] }) {
|
||||||
|
const sql = typeof query === "string" ? query : query.sql;
|
||||||
|
const args = (
|
||||||
|
typeof query === "string" ? [] : (query.args ?? [])
|
||||||
|
) as never[];
|
||||||
|
const statement = db.query(sql);
|
||||||
|
if (/^\s*(select|pragma|with)\b/i.test(sql)) {
|
||||||
|
return { rows: statement.all(...args) as unknown[], rowsAffected: 0 };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
rows: [] as unknown[],
|
||||||
|
rowsAffected: statement.run(...args).changes
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const LEGACY_NOOK = "/api/the-nook/download";
|
||||||
|
|
||||||
|
const trpcClick = (asset: string) =>
|
||||||
|
`/api/trpc/downloads.getDownloadUrl?batch=1&input=%7B%220%22%3A%7B%22asset_name%22%3A%22${asset}%22%7D%7D`;
|
||||||
|
|
||||||
|
/** Columns whose value varies per row; everything else stays NULL. */
|
||||||
|
const fixedColumns: Record<string, string> = {
|
||||||
|
method: "GET",
|
||||||
|
referrer: "https://gaze.freno.me/",
|
||||||
|
user_agent: "Mozilla/5.0 (Macintosh)",
|
||||||
|
ip_address: "203.0.113.7",
|
||||||
|
country: "GB"
|
||||||
|
};
|
||||||
|
|
||||||
|
function insertLegacy(id: string, path: string) {
|
||||||
|
const values = VISITOR_ANALYTICS_COLUMNS.map((column) => {
|
||||||
|
if (column === "id") return id;
|
||||||
|
if (column === "path") return path;
|
||||||
|
return fixedColumns[column] ?? null;
|
||||||
|
});
|
||||||
|
db.query(VISITOR_ANALYTICS_INSERT).run(...(values as never[]));
|
||||||
|
}
|
||||||
|
|
||||||
|
function canonicalPaths(): string[] {
|
||||||
|
return db
|
||||||
|
.query<{ path: string }, []>(
|
||||||
|
`SELECT path FROM VisitorAnalytics WHERE path LIKE '/api/downloads/%' ORDER BY path`
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
.map((r: { path: string }) => r.path);
|
||||||
|
}
|
||||||
|
|
||||||
|
function legacyRowCount(): number {
|
||||||
|
return db
|
||||||
|
.query<
|
||||||
|
{ n: number },
|
||||||
|
[string]
|
||||||
|
>(`SELECT COUNT(*) AS n FROM VisitorAnalytics WHERE path = ?`)
|
||||||
|
.get(LEGACY_NOOK)!.n;
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
db = new Database(":memory:");
|
||||||
|
db.exec(VISITOR_ANALYTICS_DDL);
|
||||||
|
for (const index of VISITOR_ANALYTICS_INDEXES) db.exec(index);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("backfillDownloads", () => {
|
||||||
|
it("derives a canonical row from each recoverable kind of history", async () => {
|
||||||
|
insertLegacy("nook-1", LEGACY_NOOK);
|
||||||
|
insertLegacy("gaze-1", trpcClick("gaze"));
|
||||||
|
insertLegacy("halo-1", trpcClick("inputhalo"));
|
||||||
|
insertLegacy("page-1", "/nook/");
|
||||||
|
|
||||||
|
const added = Object.fromEntries(
|
||||||
|
(await backfillDownloads(conn)).map((r) => [r.asset, r.added])
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(added).toEqual({ thenook: 1, gaze: 1, inputhalo: 1 });
|
||||||
|
expect(canonicalPaths()).toEqual([
|
||||||
|
"/api/downloads/gaze",
|
||||||
|
"/api/downloads/inputhalo",
|
||||||
|
"/api/downloads/thenook"
|
||||||
|
]);
|
||||||
|
// A page visit is not download history and gains no derived row.
|
||||||
|
expect(
|
||||||
|
db
|
||||||
|
.query(`SELECT * FROM VisitorAnalytics WHERE id = 'backfill-page-1'`)
|
||||||
|
.all()
|
||||||
|
).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the visitor detail of the row it derives from", async () => {
|
||||||
|
insertLegacy("nook-1", LEGACY_NOOK);
|
||||||
|
|
||||||
|
await backfillDownloads(conn);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
db
|
||||||
|
.query<
|
||||||
|
{ country: string; referrer: string },
|
||||||
|
[]
|
||||||
|
>(`SELECT country, referrer FROM VisitorAnalytics WHERE id = 'backfill-nook-1'`)
|
||||||
|
.get()
|
||||||
|
).toEqual({ country: "GB", referrer: "https://gaze.freno.me/" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is idempotent and never rewrites the source rows", async () => {
|
||||||
|
insertLegacy("nook-1", LEGACY_NOOK);
|
||||||
|
|
||||||
|
await backfillDownloads(conn);
|
||||||
|
const second = await backfillDownloads(conn);
|
||||||
|
|
||||||
|
expect(second).toEqual([]);
|
||||||
|
expect(canonicalPaths()).toHaveLength(1);
|
||||||
|
expect(legacyRowCount()).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("derives nothing when there is no legacy history", async () => {
|
||||||
|
expect(await backfillDownloads(conn)).toEqual([]);
|
||||||
|
expect(canonicalPaths()).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("assetFromLegacyPath", () => {
|
||||||
|
it("maps the Nook door to its canonical asset", () => {
|
||||||
|
expect(assetFromLegacyPath(LEGACY_NOOK)).toBe("thenook");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads the asset out of an encoded or a plain tRPC request URL", () => {
|
||||||
|
expect(assetFromLegacyPath(trpcClick("gaze"))).toBe("gaze");
|
||||||
|
expect(
|
||||||
|
assetFromLegacyPath(
|
||||||
|
'/api/trpc/downloads.getDownloadUrl?batch=1&input={"0":{"asset_name":"lineage"}}'
|
||||||
|
)
|
||||||
|
).toBe("lineage");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null when there is nothing to derive", () => {
|
||||||
|
expect(assetFromLegacyPath("/nook/")).toBeNull();
|
||||||
|
expect(
|
||||||
|
assetFromLegacyPath("/api/trpc/downloads.getDownloadUrl")
|
||||||
|
).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("summariseDownloads", () => {
|
||||||
|
it("labels each row by the asset in its path and totals them", () => {
|
||||||
|
expect(
|
||||||
|
summariseDownloads([
|
||||||
|
{ path: "/api/downloads/gaze", count: 4 },
|
||||||
|
{ path: "/api/downloads/thenook", count: 1 }
|
||||||
|
])
|
||||||
|
).toEqual({
|
||||||
|
total: 5,
|
||||||
|
byAsset: [
|
||||||
|
{ asset: "gaze", count: 4 },
|
||||||
|
{ asset: "thenook", count: 1 }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports nothing rather than throwing on an empty table", () => {
|
||||||
|
expect(summariseDownloads([])).toEqual({ total: 0, byAsset: [] });
|
||||||
|
});
|
||||||
|
});
|
||||||
95
src/server/backfill-downloads.ts
Normal file
95
src/server/backfill-downloads.ts
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
import { VISITOR_ANALYTICS_COLUMNS } from "~/server/analytics-schema";
|
||||||
|
import { downloadPath } from "~/lib/download-asset";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Backfill canonical `downloadPath(asset)` rows from download history that was
|
||||||
|
* recorded before `trackDownload` existed.
|
||||||
|
*
|
||||||
|
* Two shapes of history are recoverable, and only these two:
|
||||||
|
*
|
||||||
|
* - the Nook door's own rows (`/api/the-nook/download`), and
|
||||||
|
* - tRPC clicks, where the asset name still sits in the request URL as
|
||||||
|
* `?input={"0":{"asset_name":"gaze"}}` (percent-encoded in practice).
|
||||||
|
*
|
||||||
|
* Every other door wrote nothing, so there is nothing to derive for them.
|
||||||
|
*
|
||||||
|
* Derived rows keep the source row's timestamp, visitor, campaign and country,
|
||||||
|
* get `backfill-<source id>` as primary key and are written with
|
||||||
|
* `INSERT OR IGNORE` — so a second run adds nothing, and the original rows are
|
||||||
|
* left untouched (they are genuine API traffic, and the analytics page only
|
||||||
|
* counts the canonical path).
|
||||||
|
*/
|
||||||
|
|
||||||
|
const LEGACY_NOOK_PATH = "/api/the-nook/download";
|
||||||
|
const TRPC_DOOR_PREFIX = "/api/trpc/downloads.getDownloadUrl";
|
||||||
|
|
||||||
|
/** The two libsql calls used here, narrow enough for an in-memory test client. */
|
||||||
|
export interface DownloadBackfillConn {
|
||||||
|
execute(query: {
|
||||||
|
sql: string;
|
||||||
|
// `any[]` not `unknown[]`: the libsql client types its bind args as InArg[],
|
||||||
|
// and a narrower array type here would reject the real connection.
|
||||||
|
args?: any[];
|
||||||
|
}): Promise<{ rows: unknown[]; rowsAffected?: number }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DownloadBackfillResult {
|
||||||
|
asset: string;
|
||||||
|
added: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One statement: copy every column of the source row, override id and path. */
|
||||||
|
const INSERT_DERIVED = `INSERT OR IGNORE INTO VisitorAnalytics (${VISITOR_ANALYTICS_COLUMNS.join(", ")})
|
||||||
|
SELECT ${VISITOR_ANALYTICS_COLUMNS.map((column) =>
|
||||||
|
column === "id"
|
||||||
|
? "'backfill-' || id"
|
||||||
|
: column === "path"
|
||||||
|
? "?"
|
||||||
|
: `"${column}"`
|
||||||
|
).join(", ")}
|
||||||
|
FROM VisitorAnalytics
|
||||||
|
WHERE id = ?`;
|
||||||
|
|
||||||
|
/** Pull the asset name back out of a logged tRPC request URL. */
|
||||||
|
export function assetFromLegacyPath(path: string): string | null {
|
||||||
|
if (path === LEGACY_NOOK_PATH) return "thenook";
|
||||||
|
|
||||||
|
let decoded = path;
|
||||||
|
try {
|
||||||
|
decoded = decodeURIComponent(path);
|
||||||
|
} catch {
|
||||||
|
// A malformed escape is not a reason to drop the row — match on the raw.
|
||||||
|
}
|
||||||
|
|
||||||
|
return /"asset_name"\s*:\s*"([^"]+)"/.exec(decoded)?.[1] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function backfillDownloads(
|
||||||
|
conn: DownloadBackfillConn
|
||||||
|
): Promise<DownloadBackfillResult[]> {
|
||||||
|
const legacy = await conn.execute({
|
||||||
|
sql: `SELECT id, path FROM VisitorAnalytics WHERE path = ? OR path LIKE ?`,
|
||||||
|
args: [LEGACY_NOOK_PATH, `${TRPC_DOOR_PREFIX}%`]
|
||||||
|
});
|
||||||
|
|
||||||
|
const added = new Map<string, number>();
|
||||||
|
|
||||||
|
for (const row of legacy.rows) {
|
||||||
|
const { id, path } = row as { id: string; path: string };
|
||||||
|
const asset = assetFromLegacyPath(path);
|
||||||
|
if (!asset) continue;
|
||||||
|
|
||||||
|
const result = await conn.execute({
|
||||||
|
sql: INSERT_DERIVED,
|
||||||
|
args: [downloadPath(asset), id]
|
||||||
|
});
|
||||||
|
|
||||||
|
if (result.rowsAffected) {
|
||||||
|
added.set(asset, (added.get(asset) ?? 0) + result.rowsAffected);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [...added.entries()]
|
||||||
|
.map(([asset, count]) => ({ asset, added: count }))
|
||||||
|
.sort((a, b) => b.added - a.added);
|
||||||
|
}
|
||||||
61
src/server/track-download.ts
Normal file
61
src/server/track-download.ts
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
import type { APIEvent } from "@solidjs/start/server";
|
||||||
|
import { getRequestIP } from "vinxi/http";
|
||||||
|
import {
|
||||||
|
logVisit,
|
||||||
|
enrichAnalyticsEntry,
|
||||||
|
flushAnalytics
|
||||||
|
} from "~/server/analytics";
|
||||||
|
import { parseUtm } from "~/server/utm";
|
||||||
|
import { downloadPath } from "~/lib/download-asset";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record one download event under the canonical path `/api/downloads/<asset>`.
|
||||||
|
*
|
||||||
|
* Every download door funnels through here so the analytics page has a single
|
||||||
|
* label per asset to group on: the tRPC click proxy, the S3 file route that
|
||||||
|
* serves Sparkle updates, and the Nook door.
|
||||||
|
*
|
||||||
|
* Best-effort by design — a failed analytics write must never block a
|
||||||
|
* download. Flushed rather than merely buffered: a download is low-volume and
|
||||||
|
* the 30s batch timer is not guaranteed to survive the response on serverless.
|
||||||
|
*
|
||||||
|
* `event.request` is expected to be a Fetch `Request` (true for Vinxi route
|
||||||
|
* handlers and browser tRPC calls). A server-side `createCaller` hands over a
|
||||||
|
* Node request, whose headers have no `get` — that throw is swallowed below,
|
||||||
|
* so such a click is not recorded. No server-side caller downloads today.
|
||||||
|
*/
|
||||||
|
export async function trackDownload(
|
||||||
|
event: APIEvent,
|
||||||
|
asset: string
|
||||||
|
): Promise<void> {
|
||||||
|
const { request } = event;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const referrer =
|
||||||
|
request.headers.get("referer") ?? request.headers.get("referrer");
|
||||||
|
const forwarded = request.headers.get("x-forwarded-for");
|
||||||
|
|
||||||
|
await logVisit(
|
||||||
|
enrichAnalyticsEntry({
|
||||||
|
path: downloadPath(asset),
|
||||||
|
method: "GET",
|
||||||
|
referrer,
|
||||||
|
userAgent: request.headers.get("user-agent"),
|
||||||
|
ipAddress:
|
||||||
|
getRequestIP(event.nativeEvent) ??
|
||||||
|
(forwarded ? forwarded.split(",")[0]!.trim() : null),
|
||||||
|
// The click carries its own tags when a campaign links straight at the
|
||||||
|
// door; otherwise the landing page's tags ride along in the referer.
|
||||||
|
...parseUtm(request.url, referrer),
|
||||||
|
// Edge header country, overridable with ?c= for manual verification.
|
||||||
|
country:
|
||||||
|
new URL(request.url).searchParams.get("c") ??
|
||||||
|
request.headers.get("x-vercel-ip-country") ??
|
||||||
|
request.headers.get("cf-ipcountry")
|
||||||
|
})
|
||||||
|
);
|
||||||
|
await flushAnalytics();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Failed to log download of ${asset}:`, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user