feat(run): resume-aware per-check runs, verify hooks, run-state hardening
/pygienium-<check> is now resume-aware (terminal checks skipped unless --fresh) and shares run-state with all/resume; every check gets a verify hook that fails loudly when a sub-agent returns ok with no artifact; run-state clears stale errors on retry success and reconciles a run as failed only when every check failed. Drops the superseded hygiene-state.ts model.
This commit is contained in:
147
tests/verify-hooks.test.ts
Normal file
147
tests/verify-hooks.test.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* verify-hooks.test.ts — proves every artifact-producing check fails loudly
|
||||
* when its sub-agent returns ok without writing findings.md.
|
||||
*
|
||||
* This is the exact failure mode the MagniFluo run exposed: complexity,
|
||||
* deep-modules, and defensive-guards returned ok with empty text in
|
||||
* milliseconds, produced no findings.md, and — because they had no `verify`
|
||||
* hook — were stamped `complete` by the fallback gate re-run. todos was the
|
||||
* only one that failed, solely because it already had a verify hook.
|
||||
*
|
||||
* Each check now carries a `verify` hook asserting its artifacts landed. A
|
||||
* no-op agent runner (ok + empty text + no writes) must fail at verify with a
|
||||
* message naming the missing findings.md, and the check status must be
|
||||
* `failed` — never `complete`.
|
||||
*/
|
||||
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
clearChecks,
|
||||
registerCheck,
|
||||
type CheckDefinition,
|
||||
} from "../src/checks/registry.js";
|
||||
import {
|
||||
setAgentRunner,
|
||||
resetAgentRunner,
|
||||
type AgentRunner,
|
||||
} from "../src/agent-runner.js";
|
||||
import { handleCheckCommand, type PygieniumCtx } from "../src/commands.js";
|
||||
import { loadRunState } from "../src/run-state.js";
|
||||
import { complexityCheck } from "../src/checks/complexity.js";
|
||||
import { deadCodeCheck } from "../src/checks/dead-code.js";
|
||||
import { deepModulesCheck } from "../src/checks/deep-modules.js";
|
||||
import { defensiveGuardsCheck } from "../src/checks/defensive-guards.js";
|
||||
|
||||
/** Agent runner that simulates the MagniFluo bug: ok, empty, no writes. */
|
||||
const noopRunner: AgentRunner = async () => ({
|
||||
ok: true,
|
||||
text: "",
|
||||
toolCalls: [],
|
||||
});
|
||||
|
||||
function stubCtx(cwd: string): PygieniumCtx {
|
||||
return { cwd, mode: "print", hasUI: false, ui: undefined } as PygieniumCtx;
|
||||
}
|
||||
|
||||
describe("verify hooks fail loudly on empty agent output", () => {
|
||||
let cwd: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
clearChecks();
|
||||
setAgentRunner(noopRunner);
|
||||
cwd = await mkdtemp(join(tmpdir(), "pygienium-verify-"));
|
||||
// Seed one source file so the source-file gates (deep-modules,
|
||||
// defensive-guards, dead-code) pass and the check reaches analysis.
|
||||
await writeFile(join(cwd, "sample.ts"), "export const x = 1;\n", "utf8");
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetAgentRunner();
|
||||
await rm(cwd, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
/**
|
||||
* Run a check with the no-op runner and assert it fails at verify — for
|
||||
* checks where the sub-agent (not the task builder) is responsible for
|
||||
* writing findings.md.
|
||||
*/
|
||||
async function assertFailsVerify(
|
||||
check: CheckDefinition,
|
||||
findingsNeedle: string,
|
||||
): Promise<void> {
|
||||
registerCheck(check);
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
const state = await loadRunState(cwd);
|
||||
const entry = state?.checks[check.name];
|
||||
expect(entry).toBeDefined();
|
||||
expect(entry?.status).toBe("failed");
|
||||
expect(entry?.error).toContain("verify");
|
||||
expect(entry?.error).toContain("findings.md");
|
||||
// The verify phase itself is marked failed (not analysis).
|
||||
const verifyPhase = entry?.phases.find((p) => p.id === "verify");
|
||||
expect(verifyPhase?.status).toBe("failed");
|
||||
expect(verifyPhase?.error).toContain(findingsNeedle);
|
||||
// Analysis reported ok (the bug: ok + empty), but no findings captured.
|
||||
const analysisPhase = entry?.phases.find((p) => p.id === "analysis");
|
||||
expect(analysisPhase?.status).toBe("complete");
|
||||
expect(entry?.findings).toBe("");
|
||||
}
|
||||
|
||||
it("complexity fails verify when findings.md is missing", async () => {
|
||||
await assertFailsVerify(complexityCheck, "complexity verify");
|
||||
});
|
||||
|
||||
it("deep-modules fails verify when findings.md is missing", async () => {
|
||||
await assertFailsVerify(deepModulesCheck, "deep-modules verify");
|
||||
});
|
||||
|
||||
it("defensive-guards fails verify when findings.md is missing", async () => {
|
||||
await assertFailsVerify(defensiveGuardsCheck, "defensive-guards verify");
|
||||
});
|
||||
|
||||
/**
|
||||
* dead-code is hybrid: its `buildDeadCodeScanTask` deterministically
|
||||
* writes findings.md via a pre-scan BEFORE the agent runs. So a no-op
|
||||
* agent still leaves the artifact, and verify correctly passes — proving
|
||||
* the hook does not false-positive on dead-code's robust design. The
|
||||
* grep on the verify hook is still live: delete the pre-written file and
|
||||
* the same hook fails (asserted in the --fix case below for changes.md).
|
||||
*/
|
||||
it("dead-code verify passes with a no-op agent (deterministic pre-scan wrote findings.md)", async () => {
|
||||
registerCheck(deadCodeCheck);
|
||||
await handleCheckCommand(deadCodeCheck, "", stubCtx(cwd));
|
||||
const state = await loadRunState(cwd);
|
||||
const entry = state?.checks["dead-code"];
|
||||
expect(entry?.status).toBe("complete");
|
||||
const verifyPhase = entry?.phases.find((p) => p.id === "verify");
|
||||
expect(verifyPhase?.status).toBe("complete");
|
||||
// The findings.md the pre-scan wrote is on disk.
|
||||
const { stat } = await import("node:fs/promises");
|
||||
const { findingsPath } = await import("../src/checks/dead-code.js");
|
||||
await expect(stat(findingsPath(cwd))).resolves.toBeTruthy();
|
||||
});
|
||||
|
||||
it("with --fix, a missing changes.md fails verify even when findings.md exists", async () => {
|
||||
// Defensive-guards: write findings.md ourselves so the findings check
|
||||
// passes, but leave changes.md absent — verify must still fail.
|
||||
const { mkdir, writeFile: wf } = await import("node:fs/promises");
|
||||
const { dirname } = await import("node:path");
|
||||
const { findingsPath } = await import("../src/checks/defensive-guards.js");
|
||||
const f = findingsPath(cwd);
|
||||
await mkdir(dirname(f), { recursive: true });
|
||||
await wf(f, "# findings\n", "utf8");
|
||||
|
||||
registerCheck(defensiveGuardsCheck);
|
||||
// Runner writes changes.md content into its text but never to disk.
|
||||
setAgentRunner(async () => ({ ok: true, text: "", toolCalls: [] }));
|
||||
await handleCheckCommand(defensiveGuardsCheck, "--fix", stubCtx(cwd));
|
||||
|
||||
const state = await loadRunState(cwd);
|
||||
const entry = state?.checks["defensive-guards"];
|
||||
expect(entry?.status).toBe("failed");
|
||||
expect(entry?.error).toContain("changes.md");
|
||||
expect(entry?.error).toContain("verify");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user