pygienium/tests/verify-hooks.test.ts

/**
 * verify-hooks.test.ts — proves every artifact-producing check fails loudly
 * when its sub-agent returns ok without writing findings.md.
 *
 * This is the exact failure mode the MagniFluo run exposed: complexity,
 * deep-modules, and defensive-guards returned ok with empty text in
 * milliseconds, produced no findings.md, and — because they had no `verify`
 * hook — were stamped `complete` by the fallback gate re-run. todos was the
 * only one that failed, solely because it already had a verify hook.
 *
 * Each check now carries a `verify` hook asserting its artifacts landed. A
 * no-op agent runner (ok + empty text + no writes) must fail at verify with a
 * message naming the missing findings.md, and the check status must be
 * `failed` — never `complete`.
 */
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
	clearChecks,
	registerCheck,
	type CheckDefinition,
} from "../src/checks/registry.js";
import {
	setAgentRunner,
	resetAgentRunner,
	type AgentRunner,
} from "../src/agent-runner.js";
import { handleCheckCommand, type PygieniumCtx } from "../src/commands.js";
import { loadRunState } from "../src/run-state.js";
import { complexityCheck } from "../src/checks/complexity.js";
import { deadCodeCheck } from "../src/checks/dead-code.js";
import { deepModulesCheck } from "../src/checks/deep-modules.js";
import { defensiveGuardsCheck } from "../src/checks/defensive-guards.js";

/** Agent runner that simulates the MagniFluo bug: ok, empty, no writes. */
const noopRunner: AgentRunner = async () => ({
	ok: true,
	text: "",
	toolCalls: [],
});

function stubCtx(cwd: string): PygieniumCtx {
	return { cwd, mode: "print", hasUI: false, ui: undefined } as PygieniumCtx;
}

describe("verify hooks fail loudly on empty agent output", () => {
	let cwd: string;

	beforeEach(async () => {
		clearChecks();
		setAgentRunner(noopRunner);
		cwd = await mkdtemp(join(tmpdir(), "pygienium-verify-"));
		// Seed one source file so the source-file gates (deep-modules,
		// defensive-guards, dead-code) pass and the check reaches analysis.
		await writeFile(join(cwd, "sample.ts"), "export const x = 1;\n", "utf8");
	});

	afterEach(async () => {
		resetAgentRunner();
		await rm(cwd, { recursive: true, force: true }).catch(() => {});
	});

	/**
	 * Run a check with the no-op runner and assert it fails at verify — for
	 * checks where the sub-agent (not the task builder) is responsible for
	 * writing findings.md.
	 */
	async function assertFailsVerify(
		check: CheckDefinition,
		findingsNeedle: string,
	): Promise<void> {
		registerCheck(check);
		await handleCheckCommand(check, "", stubCtx(cwd));
		const state = await loadRunState(cwd);
		const entry = state?.checks[check.name];
		expect(entry).toBeDefined();
		expect(entry?.status).toBe("failed");
		expect(entry?.error).toContain("verify");
		expect(entry?.error).toContain("findings.md");
		// The verify phase itself is marked failed (not analysis).
		const verifyPhase = entry?.phases.find((p) => p.id === "verify");
		expect(verifyPhase?.status).toBe("failed");
		expect(verifyPhase?.error).toContain(findingsNeedle);
		// Analysis reported ok (the bug: ok + empty), but no findings captured.
		const analysisPhase = entry?.phases.find((p) => p.id === "analysis");
		expect(analysisPhase?.status).toBe("complete");
		expect(entry?.findings).toBe("");
	}

	it("complexity fails verify when findings.md is missing", async () => {
		await assertFailsVerify(complexityCheck, "complexity verify");
	});

	it("deep-modules fails verify when findings.md is missing", async () => {
		await assertFailsVerify(deepModulesCheck, "deep-modules verify");
	});

	it("defensive-guards fails verify when findings.md is missing", async () => {
		await assertFailsVerify(defensiveGuardsCheck, "defensive-guards verify");
	});

	/**
	 * dead-code is hybrid: its `buildDeadCodeScanTask` deterministically
	 * writes findings.md via a pre-scan BEFORE the agent runs. So a no-op
	 * agent still leaves the artifact, and verify correctly passes — proving
	 * the hook does not false-positive on dead-code's robust design. The
	 * grep on the verify hook is still live: delete the pre-written file and
	 * the same hook fails (asserted in the --fix case below for changes.md).
	 */
	it("dead-code verify passes with a no-op agent (deterministic pre-scan wrote findings.md)", async () => {
		registerCheck(deadCodeCheck);
		await handleCheckCommand(deadCodeCheck, "", stubCtx(cwd));
		const state = await loadRunState(cwd);
		const entry = state?.checks["dead-code"];
		expect(entry?.status).toBe("complete");
		const verifyPhase = entry?.phases.find((p) => p.id === "verify");
		expect(verifyPhase?.status).toBe("complete");
		// The findings.md the pre-scan wrote is on disk.
		const { stat } = await import("node:fs/promises");
		const { findingsPath } = await import("../src/checks/dead-code.js");
		await expect(stat(findingsPath(cwd))).resolves.toBeTruthy();
	});

	it("with --fix, a missing changes.md fails verify even when findings.md exists", async () => {
		// Defensive-guards: write findings.md ourselves so the findings check
		// passes, but leave changes.md absent — verify must still fail.
		const { mkdir, writeFile: wf } = await import("node:fs/promises");
		const { dirname } = await import("node:path");
		const { findingsPath } = await import("../src/checks/defensive-guards.js");
		const f = findingsPath(cwd);
		await mkdir(dirname(f), { recursive: true });
		await wf(f, "# findings\n", "utf8");

		registerCheck(defensiveGuardsCheck);
		// Runner writes changes.md content into its text but never to disk.
		setAgentRunner(async () => ({ ok: true, text: "", toolCalls: [] }));
		await handleCheckCommand(defensiveGuardsCheck, "--fix", stubCtx(cwd));

		const state = await loadRunState(cwd);
		const entry = state?.checks["defensive-guards"];
		expect(entry?.status).toBe("failed");
		expect(entry?.error).toContain("changes.md");
		expect(entry?.error).toContain("verify");
	});
});