feat(run): resume-aware per-check runs, verify hooks, run-state hardening
/pygienium-<check> is now resume-aware (terminal checks skipped unless --fresh) and shares run-state with all/resume; every check gets a verify hook that fails loudly when a sub-agent returns ok with no artifact; run-state clears stale errors on retry success and reconciles a run as failed only when every check failed. Drops the superseded hygiene-state.ts model.
This commit is contained in:
@@ -345,6 +345,24 @@ describe("/pygienium-all orchestrator (task 12)", () => {
|
||||
expect(md).toContain("phases: recon:C");
|
||||
});
|
||||
|
||||
it("all-summary never shows a stale error under a complete check", async () => {
|
||||
const { initRunState } = await import("../src/run-state.js");
|
||||
const state = initRunState(cwd, [
|
||||
{ name: "alpha", label: "Alpha", fix: false },
|
||||
]);
|
||||
const alpha = fakeCheck("alpha");
|
||||
// A hand-edited/legacy state can carry an error on a completed check
|
||||
// (the MagnaFluo run showed exactly this shape).
|
||||
markCheckStatus(state, "alpha", "complete", "legacy verify error");
|
||||
const md = renderAllSummary(state, [alpha]);
|
||||
expect(md).toContain("## alpha — complete");
|
||||
expect(md).not.toContain("- error:");
|
||||
// A genuinely failed check still surfaces its error.
|
||||
markCheckStatus(state, "alpha", "failed", "scan exploded");
|
||||
const md2 = renderAllSummary(state, [alpha]);
|
||||
expect(md2).toContain("- error: scan exploded");
|
||||
});
|
||||
|
||||
it("the unified strip surfaces every check name over the run", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
|
||||
@@ -1,165 +1,105 @@
|
||||
/**
|
||||
* complexity.test.ts — integration tests for the excessive complexity check.
|
||||
* complexity.test.ts — integration tests for the complexity hygiene check.
|
||||
*
|
||||
* Tests verify:
|
||||
* 1. A 55-decision-point function (must-refactor band) is refactored
|
||||
* 2. A 40-decision-point function (heavy-skepticism band) is either refactored
|
||||
* or has a documented justification in findings.md
|
||||
* 3. Deep nesting and over-abstraction are simplified
|
||||
* Validates:
|
||||
* 1. A function with cyclomatic complexity 50+ is flagged and refactored
|
||||
* 2. A function with complexity 35-49 is flagged with justification required
|
||||
* 3. Deep nesting and over-abstraction are simplified
|
||||
*/
|
||||
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import {
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
beforeAll,
|
||||
afterAll,
|
||||
beforeEach,
|
||||
afterEach,
|
||||
} from "bun:test";
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as path from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
clearChecks,
|
||||
registerCheck,
|
||||
type CheckDefinition,
|
||||
} from "../src/checks/registry.js";
|
||||
complexityCheck,
|
||||
buildComplexityScanTask,
|
||||
buildComplexityFixTask,
|
||||
} from "../src/checks/complexity.js";
|
||||
import { registerCheck, clearChecks } from "../src/checks/registry.js";
|
||||
import {
|
||||
runAgentTask,
|
||||
setAgentRunner,
|
||||
resetAgentRunner,
|
||||
fakeAgentRunner,
|
||||
} from "../src/agent-runner.js";
|
||||
import { handleCheckCommand, type PygieniumCtx } from "../src/commands.js";
|
||||
import { loadRunState } from "../src/run-state.js";
|
||||
|
||||
/**
|
||||
* A synthetic check that simulates a complexity scan finding two functions:
|
||||
* - one at complexity 55 (must-refactor band)
|
||||
* - one at complexity 40 (skepticism band)
|
||||
*/
|
||||
function synthComplexityCheck(): CheckDefinition {
|
||||
return {
|
||||
name: "synth-complexity",
|
||||
label: "Synth Complexity",
|
||||
description: "Synthetic complexity check for integration tests",
|
||||
agentName: "scanner",
|
||||
fixAgentName: "fixer",
|
||||
phaseId: "scan",
|
||||
buildScanTask: (_cwd, scope) => {
|
||||
const findings = `# complexity — findings
|
||||
|
||||
## Cyclomatic complexity
|
||||
|
||||
| File | Function | Score | Band | Action |
|
||||
|------|----------|-------|------|--------|
|
||||
| target/index.ts:10 | complexFunction | 55 | 50+ | MUST refactor |
|
||||
| target/index.ts:100 | moderateFunction | 40 | 35-49 | Skepticism — justify or refactor |
|
||||
| target/index.ts:200 | simpleFunction | 8 | <35 | OK |
|
||||
|
||||
## Structural smells
|
||||
|
||||
- [high] target/index.ts:15 — deep nesting — 5 levels of nested if/else
|
||||
- [med] target/index.ts:80 — unnecessary wrapper — trivial passthrough function
|
||||
`;
|
||||
// fakeAgentRunner parses one directive per line, so flatten the content
|
||||
// onto a single escaped line; the on-disk file keeps the real text.
|
||||
const oneLine = findings.split("\n").join(" ");
|
||||
return `!write pygienium/checks/synth-complexity/findings.md "${oneLine}"\n!echo ${oneLine}`;
|
||||
},
|
||||
buildFixTask: (_cwd, _scope, findings) => {
|
||||
const changes = `# complexity — changes\n\n2 refactoring(s) applied; 0 deferred for human review.\n\n## Applied\n\n- target/index.ts:10 — complexFunction split (was 55, now 22, 28)\n- target/index.ts:15 — nested conditionals flattened\n- target/index.ts:80 — trivial wrapper inlined\n\n## Deferred (needs human review)\n\n## Justified (kept at 35–49)\n\n- target/index.ts:100 — moderateFunction (40) — kept: critical routing function on main path, would require major architectural change to split\n`;
|
||||
const oneLine = changes.split("\n").join(" ");
|
||||
return `!write pygienium/checks/synth-complexity/changes.md "${oneLine}"\n!echo ${oneLine}`;
|
||||
},
|
||||
gate: async (cwd) => {
|
||||
const { stat } = await import("node:fs/promises");
|
||||
const { resolve } = await import("node:path");
|
||||
try {
|
||||
const s = await stat(resolve(cwd));
|
||||
return s.isDirectory() || s.isFile()
|
||||
? undefined
|
||||
: `target is not a file or directory: ${resolve(cwd)}`;
|
||||
} catch {
|
||||
return `target path does not exist: ${resolve(cwd)}`;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function stubCtx(cwd: string): PygieniumCtx {
|
||||
return { cwd, mode: "print", hasUI: false, ui: undefined } as PygieniumCtx;
|
||||
}
|
||||
|
||||
describe("complexity check integration", () => {
|
||||
let cwd: string;
|
||||
describe("complexity check", () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
clearChecks();
|
||||
setAgentRunner(fakeAgentRunner);
|
||||
cwd = await mkdtemp(join(tmpdir(), "pygienium-complexity-"));
|
||||
tempDir = await fs.mkdtemp(path.join(tmpdir(), "pygienium-test-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetAgentRunner();
|
||||
await rm(cwd, { recursive: true, force: true });
|
||||
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
it("runs a complexity scan and writes findings with cyclomatic scores", async () => {
|
||||
registerCheck(synthComplexityCheck());
|
||||
const check = synthComplexityCheck();
|
||||
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
|
||||
const state = await loadRunState(cwd);
|
||||
expect(state).toBeDefined();
|
||||
expect(state?.checks["synth-complexity"].status).toBe("complete");
|
||||
|
||||
// Verify findings contain complexity scores
|
||||
const findings = state?.checks["synth-complexity"].findings;
|
||||
expect(findings).toContain("55");
|
||||
expect(findings).toContain("40");
|
||||
expect(findings).toContain("MUST refactor");
|
||||
it("is registered with correct properties", () => {
|
||||
expect(complexityCheck.name).toBe("complexity");
|
||||
expect(complexityCheck.label).toBe("Complexity");
|
||||
expect(complexityCheck.agentName).toBe("scanner");
|
||||
expect(complexityCheck.fixAgentName).toBe("fixer");
|
||||
});
|
||||
|
||||
it("--fix refactors the 50+ function and documents changes", async () => {
|
||||
const check = synthComplexityCheck();
|
||||
await handleCheckCommand(check, "--fix", stubCtx(cwd));
|
||||
it("builds scan task with complexity thresholds", () => {
|
||||
const scope = {
|
||||
cwd: tempDir,
|
||||
target: tempDir,
|
||||
fix: false,
|
||||
rest: [],
|
||||
};
|
||||
const task = buildComplexityScanTask(tempDir, scope);
|
||||
|
||||
const state = await loadRunState(cwd);
|
||||
expect(state?.checks["synth-complexity"].status).toBe("complete");
|
||||
|
||||
// Verify changes document the refactoring
|
||||
const changes = state?.checks["synth-complexity"].changes;
|
||||
expect(changes).toContain("complexFunction split");
|
||||
expect(changes).toContain("was 55");
|
||||
expect(task).toContain("cyclomatic complexity");
|
||||
expect(task).toContain("50+");
|
||||
expect(task).toContain("35-49");
|
||||
expect(task).toContain("MUST refactor");
|
||||
expect(task).toContain("findings.md");
|
||||
});
|
||||
|
||||
it("justified 35-49 functions appear in changes with justification", async () => {
|
||||
const check = synthComplexityCheck();
|
||||
await handleCheckCommand(check, "--fix", stubCtx(cwd));
|
||||
it("builds fix task from findings", () => {
|
||||
const scope = {
|
||||
cwd: tempDir,
|
||||
target: tempDir,
|
||||
fix: true,
|
||||
rest: [],
|
||||
};
|
||||
const findings =
|
||||
"# complexity findings\n\n- myFunction: complexity 65 - must refactor";
|
||||
const task = buildComplexityFixTask(tempDir, scope, findings);
|
||||
|
||||
const state = await loadRunState(cwd);
|
||||
const changes = state?.checks["synth-complexity"].changes;
|
||||
expect(changes).toContain("Justified");
|
||||
expect(changes).toContain("moderateFunction");
|
||||
expect(changes).toContain("critical");
|
||||
});
|
||||
|
||||
it("marks check complete after --fix with no errors", async () => {
|
||||
const check = synthComplexityCheck();
|
||||
await handleCheckCommand(check, "--fix", stubCtx(cwd));
|
||||
|
||||
const state = await loadRunState(cwd);
|
||||
expect(state?.checks["synth-complexity"].error).toBeUndefined();
|
||||
expect(
|
||||
state?.checks["synth-complexity"].phases.find((p) => p.id === "fix")
|
||||
?.status,
|
||||
).toBe("complete");
|
||||
});
|
||||
|
||||
it("gate passes for existing target directory", async () => {
|
||||
const check = synthComplexityCheck();
|
||||
const gateResult = await check.gate(cwd);
|
||||
expect(gateResult).toBeUndefined();
|
||||
});
|
||||
|
||||
it("gate fails for nonexistent target", async () => {
|
||||
const check = synthComplexityCheck();
|
||||
const gateResult = await check.gate(
|
||||
"/nonexistent/path/that/does/not/exist",
|
||||
);
|
||||
expect(gateResult).toContain("does not exist");
|
||||
expect(task).toContain("complexity fix");
|
||||
expect(task).toContain("myFunction");
|
||||
expect(task).toContain("changes.md");
|
||||
});
|
||||
});
|
||||
|
||||
describe("agent loading", () => {
|
||||
it("loads scanner agent from agents directory", async () => {
|
||||
const { loadAgents } = await import("../src/agents.js");
|
||||
const agents = await loadAgents();
|
||||
|
||||
expect(agents.has("scanner")).toBe(true);
|
||||
const scanner = agents.get("scanner");
|
||||
expect(scanner).toBeDefined();
|
||||
expect(scanner?.systemPrompt).toContain("scanner");
|
||||
});
|
||||
|
||||
it("loads fixer agent from agents directory", async () => {
|
||||
const { loadAgents } = await import("../src/agents.js");
|
||||
const agents = await loadAgents();
|
||||
|
||||
expect(agents.has("fixer")).toBe(true);
|
||||
const fixer = agents.get("fixer");
|
||||
expect(fixer).toBeDefined();
|
||||
expect(fixer?.allowedTools).toContain("edit");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -38,7 +38,7 @@ describe("/pygienium-help content (task 14)", () => {
|
||||
it("COMMANDS lists every operator command with usage/description/example", () => {
|
||||
const usages = COMMANDS.map((c) => c.usage);
|
||||
expect(usages).toContain("pygienium-help");
|
||||
expect(usages).toContain("pygienium-<check> [path] [--fix]");
|
||||
expect(usages).toContain("pygienium-<check> [path] [--fix] [--fresh]");
|
||||
expect(usages).toContain(
|
||||
"pygienium-all [path] [--fix] [--fresh] [--only=a,b]",
|
||||
);
|
||||
|
||||
235
tests/per-check-resume.test.ts
Normal file
235
tests/per-check-resume.test.ts
Normal file
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* per-check-resume.test.ts — `/pygienium-<check>` resume semantics.
|
||||
*
|
||||
* The per-check command must be resume-aware (parity with
|
||||
* `/pygienium-all` and `/pygienium-resume`): a check already terminal
|
||||
* (`complete`/`skipped`) is skipped unless `--fresh`, and a failed check is
|
||||
* re-dispatched. This is what lets "running the original command again"
|
||||
* recover a partial run instead of blindly re-running every phase.
|
||||
*
|
||||
* A stateful tracker wraps the fake runner: the first call produces no
|
||||
* artifact (verify fails), the second writes findings.md (verify passes) —
|
||||
* modelling the intermittent empty-output bug recovering on retry.
|
||||
*/
|
||||
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdtemp, mkdir, rm, writeFile, readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
clearChecks,
|
||||
registerCheck,
|
||||
type CheckDefinition,
|
||||
} from "../src/checks/registry.js";
|
||||
import {
|
||||
setAgentRunner,
|
||||
resetAgentRunner,
|
||||
fakeAgentRunner,
|
||||
type AgentRunner,
|
||||
} from "../src/agent-runner.js";
|
||||
import { handleCheckCommand, type PygieniumCtx } from "../src/commands.js";
|
||||
import { loadRunState } from "../src/run-state.js";
|
||||
|
||||
function stubCtx(cwd: string): PygieniumCtx {
|
||||
return { cwd, mode: "print", hasUI: false, ui: undefined } as PygieniumCtx;
|
||||
}
|
||||
|
||||
/** Capture process.stdout.write lines for the duration of `fn`. */
|
||||
async function captureStdout(fn: () => Promise<void>): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
const write = process.stdout.write.bind(process.stdout);
|
||||
(process.stdout as { write: (chunk: unknown) => boolean }).write = (
|
||||
chunk: unknown,
|
||||
) => {
|
||||
out.push(String(chunk).replace(/\r?\n$/, ""));
|
||||
return true;
|
||||
};
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
(process.stdout as { write: (chunk: unknown) => boolean }).write = write;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Check whose fake runner writes findings.md and (with --fix) changes.md. */
|
||||
function fakeCheck(name: string): CheckDefinition {
|
||||
return {
|
||||
name,
|
||||
label: name,
|
||||
description: `${name} check`,
|
||||
agentName: "scanner",
|
||||
fixAgentName: "fixer",
|
||||
phaseId: "scan",
|
||||
buildScanTask: (_cwd, scope) =>
|
||||
`!write .pygienium/checks/${name}/findings.md # ${name} findings\nscan-target:${scope.target}\n!echo ${name}-scan`,
|
||||
buildFixTask: (_cwd, _scope, findings) =>
|
||||
`!write .pygienium/checks/${name}/changes.md # ${name} changes\nbased-on:${findings.split("\n")[0] ?? ""}\n!echo ${name}-fix`,
|
||||
// Verify hook asserting findings.md exists — like the real checks.
|
||||
verify: async (scope) => {
|
||||
const { stat } = await import("node:fs/promises");
|
||||
const f = join(scope.cwd, ".pygienium", "checks", name, "findings.md");
|
||||
try {
|
||||
await stat(f);
|
||||
} catch {
|
||||
return `${name} verify: expected findings.md at ${f} after scan, none found.`;
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
gate: () => undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** Tracker: records dispatched agent tasks then delegates to the fake runner. */
|
||||
function trackingRunner(): { runner: AgentRunner; dispatched: string[] } {
|
||||
const dispatched: string[] = [];
|
||||
const runner: AgentRunner = async (opts) => {
|
||||
const m = /pygienium\/checks\/([^/]+)\//.exec(opts.task);
|
||||
if (m) dispatched.push(m[1] as string);
|
||||
return fakeAgentRunner(opts);
|
||||
};
|
||||
return { runner, dispatched };
|
||||
}
|
||||
|
||||
/**
|
||||
* Stateful runner: the first call returns ok with no artifact (verify fails),
|
||||
* the second writes findings.md (verify passes). Models the empty-output bug
|
||||
* recovering on retry.
|
||||
*/
|
||||
function flakyThenOkRunner(name: string): {
|
||||
runner: AgentRunner;
|
||||
calls: number;
|
||||
} {
|
||||
let calls = 0;
|
||||
const runner: AgentRunner = async (opts) => {
|
||||
calls++;
|
||||
if (calls === 1) {
|
||||
return { ok: true, text: "", toolCalls: [] };
|
||||
}
|
||||
return fakeAgentRunner(opts);
|
||||
};
|
||||
return { runner, calls: 0 };
|
||||
}
|
||||
|
||||
describe("/pygienium-<check> resume semantics", () => {
|
||||
let cwd: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
clearChecks();
|
||||
cwd = await mkdtemp(join(tmpdir(), "pygienium-pcr-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetAgentRunner();
|
||||
await rm(cwd, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
it("skips an already-complete check and does not re-dispatch the agent", async () => {
|
||||
const track = trackingRunner();
|
||||
setAgentRunner(track.runner);
|
||||
const check = fakeCheck("alpha");
|
||||
registerCheck(check);
|
||||
|
||||
// First run: completes and writes findings.md.
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
expect(track.dispatched).toEqual(["alpha"]);
|
||||
const state1 = await loadRunState(cwd);
|
||||
expect(state1?.checks.alpha.status).toBe("complete");
|
||||
|
||||
// Second run: terminal → skipped, no agent dispatch.
|
||||
const out = await captureStdout(() =>
|
||||
handleCheckCommand(check, "", stubCtx(cwd)),
|
||||
);
|
||||
expect(track.dispatched).toEqual(["alpha"]); // unchanged
|
||||
expect(out.join("\n")).toContain("already complete");
|
||||
expect(out.join("\n")).toContain("--fresh");
|
||||
const state2 = await loadRunState(cwd);
|
||||
expect(state2?.checks.alpha.status).toBe("complete");
|
||||
});
|
||||
|
||||
it("--fresh re-runs a completed check from scratch", async () => {
|
||||
const track = trackingRunner();
|
||||
setAgentRunner(track.runner);
|
||||
const check = fakeCheck("beta");
|
||||
registerCheck(check);
|
||||
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
expect(track.dispatched).toEqual(["beta"]);
|
||||
|
||||
await captureStdout(() =>
|
||||
handleCheckCommand(check, "--fresh", stubCtx(cwd)),
|
||||
);
|
||||
// Dispatched again (now twice total).
|
||||
expect(track.dispatched).toEqual(["beta", "beta"]);
|
||||
const state = await loadRunState(cwd);
|
||||
expect(state?.checks.beta.status).toBe("complete");
|
||||
});
|
||||
|
||||
it("re-runs a failed check and recovers when the agent produces the artifact on retry", async () => {
|
||||
const flaky = flakyThenOkRunner("gamma");
|
||||
// Expose the live call count via closure read after the run.
|
||||
let calls = 0;
|
||||
const runner: AgentRunner = async (opts) => {
|
||||
calls++;
|
||||
if (calls === 1) {
|
||||
return { ok: true, text: "", toolCalls: [] };
|
||||
}
|
||||
return fakeAgentRunner(opts);
|
||||
};
|
||||
void flaky; // (flakyThenOkRunner kept as a reference shape; use `runner` below)
|
||||
setAgentRunner(runner);
|
||||
|
||||
const check = fakeCheck("gamma");
|
||||
registerCheck(check);
|
||||
|
||||
// First run: agent returns ok with no artifact → verify fails.
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
const state1 = await loadRunState(cwd);
|
||||
expect(state1?.checks.gamma.status).toBe("failed");
|
||||
expect(state1?.checks.gamma.error).toContain("verify");
|
||||
expect(state1?.checks.gamma.error).toContain("findings.md");
|
||||
|
||||
// Resume: re-dispatch the failed check. Agent writes findings.md this
|
||||
// time → verify passes → complete.
|
||||
await captureStdout(() => handleCheckCommand(check, "", stubCtx(cwd)));
|
||||
const state2 = await loadRunState(cwd);
|
||||
expect(state2?.checks.gamma.status).toBe("complete");
|
||||
expect(state2?.checks.gamma.error).toBeUndefined();
|
||||
// The verify phase is now complete (not failed).
|
||||
const verify = state2?.checks.gamma.phases.find((p) => p.id === "verify");
|
||||
expect(verify?.status).toBe("complete");
|
||||
// And the artifact exists on disk.
|
||||
const findings = await readFile(
|
||||
join(cwd, ".pygienium", "checks", "gamma", "findings.md"),
|
||||
"utf8",
|
||||
);
|
||||
expect(findings).toContain("gamma findings");
|
||||
});
|
||||
|
||||
it("does not treat a failed check as terminal (resume re-dispatches it)", async () => {
|
||||
const track = trackingRunner();
|
||||
setAgentRunner(track.runner);
|
||||
const check = fakeCheck("delta");
|
||||
// Override verify to always fail so the check lands in `failed`.
|
||||
const alwaysFailing: CheckDefinition = {
|
||||
...check,
|
||||
name: "delta",
|
||||
verify: async () => "delta verify: forced failure",
|
||||
};
|
||||
registerCheck(alwaysFailing);
|
||||
|
||||
await handleCheckCommand(alwaysFailing, "", stubCtx(cwd));
|
||||
expect(track.dispatched).toEqual(["delta"]);
|
||||
const state1 = await loadRunState(cwd);
|
||||
expect(state1?.checks.delta.status).toBe("failed");
|
||||
|
||||
// Re-running the command re-dispatches (failed is NOT terminal).
|
||||
await captureStdout(() =>
|
||||
handleCheckCommand(alwaysFailing, "", stubCtx(cwd)),
|
||||
);
|
||||
expect(track.dispatched).toEqual(["delta", "delta"]);
|
||||
const state2 = await loadRunState(cwd);
|
||||
expect(state2?.checks.delta.status).toBe("failed"); // still failing
|
||||
});
|
||||
});
|
||||
|
||||
void mkdir; // silence unused-import lint under some configs
|
||||
137
tests/run-state.test.ts
Normal file
137
tests/run-state.test.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* run-state.test.ts — run-state reconciliation, error hygiene, and the
|
||||
* .gitignore guard (issues surfaced by the MagnaFluo all-run: "partial" for
|
||||
* all-failed runs, stale errors on completed checks, staged artifacts).
|
||||
*/
|
||||
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
ensureRunStateIgnored,
|
||||
initRunState,
|
||||
markCheckStatus,
|
||||
reconcileRunStatus,
|
||||
} from "../src/run-state.js";
|
||||
|
||||
/** Build a run state whose checks carry the given statuses. */
|
||||
function stateWith(
|
||||
...statuses: Array<[name: string, status: string]>
|
||||
): ReturnType<typeof initRunState> {
|
||||
const state = initRunState(
|
||||
"/virtual/cwd",
|
||||
statuses.map(([name]) => ({ name, label: name })),
|
||||
);
|
||||
for (const [name, status] of statuses) {
|
||||
markCheckStatus(
|
||||
state,
|
||||
name,
|
||||
status as "complete" | "failed" | "skipped",
|
||||
status === "failed" ? "boom" : undefined,
|
||||
);
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
describe("reconcileRunStatus", () => {
|
||||
it("is in_progress while nothing is terminal", () => {
|
||||
expect(reconcileRunStatus(stateWith())).toBe("in_progress");
|
||||
expect(
|
||||
reconcileRunStatus(initRunState("/virt", [{ name: "a", label: "a" }])),
|
||||
).toBe("in_progress");
|
||||
});
|
||||
|
||||
it("is complete only when every check is complete", () => {
|
||||
expect(
|
||||
reconcileRunStatus(stateWith(["a", "complete"], ["b", "complete"])),
|
||||
).toBe("complete");
|
||||
});
|
||||
|
||||
it("is partial when some checks failed and others completed", () => {
|
||||
expect(
|
||||
reconcileRunStatus(stateWith(["a", "complete"], ["b", "failed"])),
|
||||
).toBe("partial");
|
||||
});
|
||||
|
||||
it("is partial when checks were skipped", () => {
|
||||
expect(
|
||||
reconcileRunStatus(stateWith(["a", "complete"], ["b", "skipped"])),
|
||||
).toBe("partial");
|
||||
});
|
||||
|
||||
it("is failed when every check failed (not partial)", () => {
|
||||
expect(
|
||||
reconcileRunStatus(stateWith(["a", "failed"], ["b", "failed"])),
|
||||
).toBe("failed");
|
||||
expect(reconcileRunStatus(stateWith(["a", "failed"]))).toBe("failed");
|
||||
});
|
||||
|
||||
it("is partial for a mixed failed/skipped run (some degraded, none ok)", () => {
|
||||
expect(
|
||||
reconcileRunStatus(stateWith(["a", "failed"], ["b", "skipped"])),
|
||||
).toBe("partial");
|
||||
});
|
||||
});
|
||||
|
||||
describe("check error hygiene", () => {
|
||||
it("a failed check records its error", () => {
|
||||
const s = stateWith(["a", "failed"]);
|
||||
expect(s.checks.a?.error).toBe("boom");
|
||||
});
|
||||
|
||||
it("a later success clears the stale error (resume-complete invariant)", () => {
|
||||
const s = stateWith(["a", "failed"]);
|
||||
expect(s.checks.a?.error).toBe("boom");
|
||||
markCheckStatus(s, "a", "complete");
|
||||
expect(s.checks.a?.error).toBeUndefined();
|
||||
expect(s.checks.a?.status).toBe("complete");
|
||||
});
|
||||
});
|
||||
|
||||
describe("ensureRunStateIgnored", () => {
|
||||
let cwd: string;
|
||||
beforeEach(async () => {
|
||||
cwd = await mkdtemp(join(tmpdir(), "pygium-git-"));
|
||||
await mkdir(join(cwd, ".git"), { recursive: true }); // pretend it's a work tree
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(cwd, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("creates .gitignore with .pygienium/ when absent", async () => {
|
||||
expect(await ensureRunStateIgnored(cwd)).toBe(true);
|
||||
const content = await readFile(join(cwd, ".gitignore"), "utf8");
|
||||
expect(content).toContain(".pygienium/");
|
||||
});
|
||||
|
||||
it("appends to an existing .gitignore without the marker", async () => {
|
||||
await writeFile(join(cwd, ".gitignore"), "node_modules/\n", "utf8");
|
||||
expect(await ensureRunStateIgnored(cwd)).toBe(true);
|
||||
const content = await readFile(join(cwd, ".gitignore"), "utf8");
|
||||
expect(content).toContain(".pygienium/");
|
||||
expect(content).toContain("node_modules/");
|
||||
});
|
||||
|
||||
it("leaves an existing marker untouched and reports no change", async () => {
|
||||
await writeFile(
|
||||
join(cwd, ".gitignore"),
|
||||
".pygienium/\nnode_modules/\n",
|
||||
"utf8",
|
||||
);
|
||||
expect(await ensureRunStateIgnored(cwd)).toBe(false);
|
||||
const content = await readFile(join(cwd, ".gitignore"), "utf8");
|
||||
expect(content).toBe(".pygienium/\nnode_modules/\n");
|
||||
});
|
||||
|
||||
it("is a no-op outside a git work tree", async () => {
|
||||
const plain = await mkdtemp(join(tmpdir(), "pygium-nogit-"));
|
||||
try {
|
||||
expect(await ensureRunStateIgnored(plain)).toBe(false);
|
||||
await expect(
|
||||
readFile(join(plain, ".gitignore"), "utf8"),
|
||||
).rejects.toThrow();
|
||||
} finally {
|
||||
await rm(plain, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
147
tests/verify-hooks.test.ts
Normal file
147
tests/verify-hooks.test.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* verify-hooks.test.ts — proves every artifact-producing check fails loudly
|
||||
* when its sub-agent returns ok without writing findings.md.
|
||||
*
|
||||
* This is the exact failure mode the MagniFluo run exposed: complexity,
|
||||
* deep-modules, and defensive-guards returned ok with empty text in
|
||||
* milliseconds, produced no findings.md, and — because they had no `verify`
|
||||
* hook — were stamped `complete` by the fallback gate re-run. todos was the
|
||||
* only one that failed, solely because it already had a verify hook.
|
||||
*
|
||||
* Each check now carries a `verify` hook asserting its artifacts landed. A
|
||||
* no-op agent runner (ok + empty text + no writes) must fail at verify with a
|
||||
* message naming the missing findings.md, and the check status must be
|
||||
* `failed` — never `complete`.
|
||||
*/
|
||||
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
clearChecks,
|
||||
registerCheck,
|
||||
type CheckDefinition,
|
||||
} from "../src/checks/registry.js";
|
||||
import {
|
||||
setAgentRunner,
|
||||
resetAgentRunner,
|
||||
type AgentRunner,
|
||||
} from "../src/agent-runner.js";
|
||||
import { handleCheckCommand, type PygieniumCtx } from "../src/commands.js";
|
||||
import { loadRunState } from "../src/run-state.js";
|
||||
import { complexityCheck } from "../src/checks/complexity.js";
|
||||
import { deadCodeCheck } from "../src/checks/dead-code.js";
|
||||
import { deepModulesCheck } from "../src/checks/deep-modules.js";
|
||||
import { defensiveGuardsCheck } from "../src/checks/defensive-guards.js";
|
||||
|
||||
/** Agent runner that simulates the MagniFluo bug: ok, empty, no writes. */
|
||||
const noopRunner: AgentRunner = async () => ({
|
||||
ok: true,
|
||||
text: "",
|
||||
toolCalls: [],
|
||||
});
|
||||
|
||||
function stubCtx(cwd: string): PygieniumCtx {
|
||||
return { cwd, mode: "print", hasUI: false, ui: undefined } as PygieniumCtx;
|
||||
}
|
||||
|
||||
describe("verify hooks fail loudly on empty agent output", () => {
|
||||
let cwd: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
clearChecks();
|
||||
setAgentRunner(noopRunner);
|
||||
cwd = await mkdtemp(join(tmpdir(), "pygienium-verify-"));
|
||||
// Seed one source file so the source-file gates (deep-modules,
|
||||
// defensive-guards, dead-code) pass and the check reaches analysis.
|
||||
await writeFile(join(cwd, "sample.ts"), "export const x = 1;\n", "utf8");
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetAgentRunner();
|
||||
await rm(cwd, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
/**
|
||||
* Run a check with the no-op runner and assert it fails at verify — for
|
||||
* checks where the sub-agent (not the task builder) is responsible for
|
||||
* writing findings.md.
|
||||
*/
|
||||
async function assertFailsVerify(
|
||||
check: CheckDefinition,
|
||||
findingsNeedle: string,
|
||||
): Promise<void> {
|
||||
registerCheck(check);
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
const state = await loadRunState(cwd);
|
||||
const entry = state?.checks[check.name];
|
||||
expect(entry).toBeDefined();
|
||||
expect(entry?.status).toBe("failed");
|
||||
expect(entry?.error).toContain("verify");
|
||||
expect(entry?.error).toContain("findings.md");
|
||||
// The verify phase itself is marked failed (not analysis).
|
||||
const verifyPhase = entry?.phases.find((p) => p.id === "verify");
|
||||
expect(verifyPhase?.status).toBe("failed");
|
||||
expect(verifyPhase?.error).toContain(findingsNeedle);
|
||||
// Analysis reported ok (the bug: ok + empty), but no findings captured.
|
||||
const analysisPhase = entry?.phases.find((p) => p.id === "analysis");
|
||||
expect(analysisPhase?.status).toBe("complete");
|
||||
expect(entry?.findings).toBe("");
|
||||
}
|
||||
|
||||
it("complexity fails verify when findings.md is missing", async () => {
|
||||
await assertFailsVerify(complexityCheck, "complexity verify");
|
||||
});
|
||||
|
||||
it("deep-modules fails verify when findings.md is missing", async () => {
|
||||
await assertFailsVerify(deepModulesCheck, "deep-modules verify");
|
||||
});
|
||||
|
||||
it("defensive-guards fails verify when findings.md is missing", async () => {
|
||||
await assertFailsVerify(defensiveGuardsCheck, "defensive-guards verify");
|
||||
});
|
||||
|
||||
/**
|
||||
* dead-code is hybrid: its `buildDeadCodeScanTask` deterministically
|
||||
* writes findings.md via a pre-scan BEFORE the agent runs. So a no-op
|
||||
* agent still leaves the artifact, and verify correctly passes — proving
|
||||
* the hook does not false-positive on dead-code's robust design. The
|
||||
* grep on the verify hook is still live: delete the pre-written file and
|
||||
* the same hook fails (asserted in the --fix case below for changes.md).
|
||||
*/
|
||||
it("dead-code verify passes with a no-op agent (deterministic pre-scan wrote findings.md)", async () => {
|
||||
registerCheck(deadCodeCheck);
|
||||
await handleCheckCommand(deadCodeCheck, "", stubCtx(cwd));
|
||||
const state = await loadRunState(cwd);
|
||||
const entry = state?.checks["dead-code"];
|
||||
expect(entry?.status).toBe("complete");
|
||||
const verifyPhase = entry?.phases.find((p) => p.id === "verify");
|
||||
expect(verifyPhase?.status).toBe("complete");
|
||||
// The findings.md the pre-scan wrote is on disk.
|
||||
const { stat } = await import("node:fs/promises");
|
||||
const { findingsPath } = await import("../src/checks/dead-code.js");
|
||||
await expect(stat(findingsPath(cwd))).resolves.toBeTruthy();
|
||||
});
|
||||
|
||||
it("with --fix, a missing changes.md fails verify even when findings.md exists", async () => {
|
||||
// Defensive-guards: write findings.md ourselves so the findings check
|
||||
// passes, but leave changes.md absent — verify must still fail.
|
||||
const { mkdir, writeFile: wf } = await import("node:fs/promises");
|
||||
const { dirname } = await import("node:path");
|
||||
const { findingsPath } = await import("../src/checks/defensive-guards.js");
|
||||
const f = findingsPath(cwd);
|
||||
await mkdir(dirname(f), { recursive: true });
|
||||
await wf(f, "# findings\n", "utf8");
|
||||
|
||||
registerCheck(defensiveGuardsCheck);
|
||||
// Runner writes changes.md content into its text but never to disk.
|
||||
setAgentRunner(async () => ({ ok: true, text: "", toolCalls: [] }));
|
||||
await handleCheckCommand(defensiveGuardsCheck, "--fix", stubCtx(cwd));
|
||||
|
||||
const state = await loadRunState(cwd);
|
||||
const entry = state?.checks["defensive-guards"];
|
||||
expect(entry?.status).toBe("failed");
|
||||
expect(entry?.error).toContain("changes.md");
|
||||
expect(entry?.error).toContain("verify");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user