402 lines
14 KiB
TypeScript
402 lines
14 KiB
TypeScript
/**
|
||
* all.test.ts — integration test for the `/pygienium-all` orchestrator (task 12).
|
||
*
|
||
* Mirrors the spec scenario: run `/pygienium-all` on a small repo and assert:
|
||
* - every registered check runs exactly once in registry order;
|
||
* - run-state shows all checks complete and the overall run complete;
|
||
* - `.pygienium/all-summary.md` is present and lists per-check outcomes;
|
||
* - `--only=alpha,gamma` narrows the candidate set preserving order;
|
||
* - interrupted/resumed runs re-dispatch non-terminal checks while skipping
|
||
* terminal ones, unless `--fresh` resets everything.
|
||
*
|
||
* A tracker wraps the fake agent runner so we can assert dispatch order and
|
||
* counts without a model.
|
||
*/
|
||
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
|
||
import { mkdtemp, mkdir, readFile, rm } from "node:fs/promises";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import {
|
||
clearChecks,
|
||
registerCheck,
|
||
type CheckDefinition,
|
||
} from "../src/checks/registry.js";
|
||
import {
|
||
setAgentRunner,
|
||
resetAgentRunner,
|
||
fakeAgentRunner,
|
||
type AgentRunner,
|
||
} from "../src/agent-runner.js";
|
||
import { handleAllCommand, type PygieniumCtx } from "../src/commands.js";
|
||
import {
|
||
parseAllArgs,
|
||
runAllChecks,
|
||
allSummaryPath,
|
||
renderAllSummary,
|
||
selectChecks,
|
||
} from "../src/modes/all.js";
|
||
import {
|
||
loadRunState,
|
||
markCheckStatus,
|
||
applyPhaseStatus,
|
||
PHASE_RECON,
|
||
PHASE_ANALYSIS,
|
||
PHASE_FIX,
|
||
PHASE_VERIFY,
|
||
PHASE_CLEANUP,
|
||
} from "../src/run-state.js";
|
||
import { writeFile } from "node:fs/promises";
|
||
|
||
/** Build a deterministic check whose fake runner writes on-disk artifacts. */
|
||
function fakeCheck(name: string): CheckDefinition {
|
||
return {
|
||
name,
|
||
label: name.charAt(0).toUpperCase() + name.slice(1),
|
||
description: `${name} check`,
|
||
agentName: "scanner",
|
||
fixAgentName: "fixer",
|
||
phaseId: "scan",
|
||
buildScanTask: (_cwd, scope) =>
|
||
`!write .pygienium/checks/${name}/findings.md # ${name} findings\nscan-target:${scope.target}\n!echo ${name}-scan`,
|
||
buildFixTask: (_cwd, _scope, findings) =>
|
||
`!write .pygienium/checks/${name}/changes.md # ${name} changes\nbased-on:${findings.split("\n")[0] ?? ""}\n!echo ${name}-fix`,
|
||
gate: () => undefined,
|
||
};
|
||
}
|
||
|
||
function stubCtx(cwd: string): PygieniumCtx {
|
||
return { cwd, hasUI: false, ui: undefined } as PygieniumCtx;
|
||
}
|
||
|
||
/** Tracker: records dispatched agent tasks then delegates to the fake runner. */
|
||
function trackingRunner(): { runner: AgentRunner; dispatched: string[] } {
|
||
const dispatched: string[] = [];
|
||
const runner: AgentRunner = async (opts) => {
|
||
const m = /pygienium\/checks\/([^/]+)\//.exec(opts.task);
|
||
if (m) dispatched.push(m[1] as string);
|
||
return fakeAgentRunner(opts);
|
||
};
|
||
return { runner, dispatched };
|
||
}
|
||
|
||
/** Capture process.stdout.write lines for the duration of `fn`. */
|
||
async function captureStdout(fn: () => Promise<void>): Promise<string[]> {
|
||
const out: string[] = [];
|
||
const write = process.stdout.write.bind(process.stdout);
|
||
(process.stdout as { write: (chunk: unknown) => boolean }).write = (
|
||
chunk: unknown,
|
||
) => {
|
||
out.push(String(chunk).replace(/\r?\n$/, ""));
|
||
return true;
|
||
};
|
||
try {
|
||
await fn();
|
||
} finally {
|
||
(process.stdout as { write: (chunk: unknown) => boolean }).write = write;
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/** Mark a check as fully complete in the run-state (helper for seeding). */
|
||
function markComplete(
|
||
state: Parameters<typeof markCheckStatus>[0],
|
||
name: string,
|
||
): void {
|
||
for (const phaseId of [
|
||
PHASE_RECON,
|
||
PHASE_ANALYSIS,
|
||
PHASE_FIX,
|
||
PHASE_VERIFY,
|
||
PHASE_CLEANUP,
|
||
]) {
|
||
applyPhaseStatus(state, name, phaseId, "complete");
|
||
}
|
||
markCheckStatus(state, name, "complete");
|
||
}
|
||
|
||
describe("/pygienium-all orchestrator (task 12)", () => {
|
||
let cwd: string;
|
||
let track: ReturnType<typeof trackingRunner>;
|
||
|
||
beforeEach(async () => {
|
||
clearChecks();
|
||
track = trackingRunner();
|
||
setAgentRunner(track.runner);
|
||
cwd = await mkdtemp(join(tmpdir(), "pygienium-all-"));
|
||
// Seed a source file so the gate passes and recon has something to scan.
|
||
await mkdir(join(cwd, "src"), { recursive: true });
|
||
});
|
||
|
||
afterEach(async () => {
|
||
resetAgentRunner();
|
||
await rm(cwd, { recursive: true, force: true });
|
||
});
|
||
|
||
it("parseAllArgs parses path, --fix, --fresh, --no-gitignore, and --only", () => {
|
||
const p = parseAllArgs("subdir --fix --only=alpha,beta --fresh", cwd);
|
||
expect(p.target).toBe(join(cwd, "subdir"));
|
||
expect(p.fix).toBe(true);
|
||
expect(p.fresh).toBe(true);
|
||
expect(p.gitignore).toBe(true); // default: keep the .gitignore guard on
|
||
expect(p.only).toEqual(["alpha", "beta"]);
|
||
const noGi = parseAllArgs("--no-gitignore", cwd);
|
||
expect(noGi.gitignore).toBe(false);
|
||
expect(noGi.target).toBe(cwd);
|
||
});
|
||
|
||
it("selectChecks preserves registry order for the --only subset", () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
registerCheck(fakeCheck("gamma"));
|
||
const subset = selectChecks(["gamma", "alpha"]); // order in --only is irrelevant
|
||
expect(subset.map((c) => c.name)).toEqual(["alpha", "gamma"]);
|
||
expect(selectChecks().length).toBe(3);
|
||
expect(selectChecks([]).length).toBe(3);
|
||
});
|
||
|
||
it("runs every registered check exactly once in registry order", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
registerCheck(fakeCheck("gamma"));
|
||
|
||
await captureStdout(() => handleAllCommand("--fix", stubCtx(cwd)));
|
||
|
||
// Each check dispatched once for scan + once for fix (3 checks × 2 phases).
|
||
const scanDispatches = track.dispatched.filter((n) => n !== undefined);
|
||
expect(scanDispatches).toEqual([
|
||
"alpha",
|
||
"alpha",
|
||
"beta",
|
||
"beta",
|
||
"gamma",
|
||
"gamma",
|
||
]);
|
||
|
||
const state = await loadRunState(cwd);
|
||
expect(state?.status).toBe("complete");
|
||
expect(state?.checks.alpha.status).toBe("complete");
|
||
expect(state?.checks.beta.status).toBe("complete");
|
||
expect(state?.checks.gamma.status).toBe("complete");
|
||
expect(state?.recon.complete).toBe(true);
|
||
});
|
||
|
||
it("writes .pygienium/all-summary.md listing per-check outcomes", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
|
||
await captureStdout(() => handleAllCommand("--fix", stubCtx(cwd)));
|
||
|
||
const summary = await readFile(allSummaryPath(cwd), "utf8");
|
||
expect(summary).toContain("# Pygienium all-run summary");
|
||
expect(summary).toContain("- status: complete");
|
||
expect(summary).toContain("## alpha — complete");
|
||
expect(summary).toContain("## beta — complete");
|
||
expect(summary).toContain("## alpha — complete (--fix)");
|
||
// Artifact paths + line counts are referenced.
|
||
expect(summary).toContain("findings:");
|
||
expect(summary).toContain("changes:");
|
||
// Artifacts actually exist on disk.
|
||
const alphaFindings = await readFile(
|
||
join(cwd, ".pygienium", "checks", "alpha", "findings.md"),
|
||
"utf8",
|
||
);
|
||
expect(alphaFindings).toContain("alpha findings");
|
||
});
|
||
|
||
it("--only narrows the run to the named subset", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
registerCheck(fakeCheck("gamma"));
|
||
|
||
await captureStdout(() =>
|
||
handleAllCommand("--only=alpha,gamma", stubCtx(cwd)),
|
||
);
|
||
|
||
// Only alpha + gamma dispatched (beta never touched).
|
||
expect(track.dispatched).toContain("alpha");
|
||
expect(track.dispatched).toContain("gamma");
|
||
expect(track.dispatched).not.toContain("beta");
|
||
|
||
const state = await loadRunState(cwd);
|
||
expect(state?.checks.alpha.status).toBe("complete");
|
||
expect(state?.checks.gamma.status).toBe("complete");
|
||
// beta was not part of the selected set, so has no entry.
|
||
expect(state?.checks.beta).toBeUndefined();
|
||
|
||
const summary = await readFile(allSummaryPath(cwd), "utf8");
|
||
expect(summary).toContain("## alpha");
|
||
expect(summary).toContain("## gamma");
|
||
expect(summary).not.toContain("## beta");
|
||
});
|
||
|
||
it("skips terminal checks on re-run; --fresh re-runs them", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
|
||
// First run: both complete.
|
||
await captureStdout(() => handleAllCommand("--fix", stubCtx(cwd)));
|
||
expect(track.dispatched.length).toBe(4); // 2 checks × 2 phases
|
||
const firstAlphaFix = await readFile(
|
||
join(cwd, ".pygienium", "checks", "alpha", "changes.md"),
|
||
"utf8",
|
||
);
|
||
|
||
// Second run without --fresh: both already terminal → skipped.
|
||
track.dispatched.length = 0;
|
||
const out = await captureStdout(() =>
|
||
handleAllCommand("--fix", stubCtx(cwd)),
|
||
);
|
||
expect(track.dispatched).toHaveLength(0);
|
||
expect(out.join("\n")).toContain("skipping");
|
||
const state2 = await loadRunState(cwd);
|
||
expect(state2?.status).toBe("complete");
|
||
|
||
// Third run with --fresh: both re-dispatched from scratch.
|
||
track.dispatched.length = 0;
|
||
await captureStdout(() => handleAllCommand("--fix --fresh", stubCtx(cwd)));
|
||
expect(track.dispatched).toEqual(["alpha", "alpha", "beta", "beta"]);
|
||
const state3 = await loadRunState(cwd);
|
||
expect(state3?.status).toBe("complete");
|
||
// The fresh re-run overwrote alpha's changes.md (still valid content).
|
||
const alphaFix2 = await readFile(
|
||
join(cwd, ".pygienium", "checks", "alpha", "changes.md"),
|
||
"utf8",
|
||
);
|
||
expect(alphaFix2).toContain("alpha changes");
|
||
void firstAlphaFix;
|
||
});
|
||
|
||
it("resumes an interrupted run (re-dispatches non-terminal checks only)", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
|
||
// Seed an interrupted run: alpha complete, beta pending (interrupted).
|
||
const { initRunState, saveRunState } = await import("../src/run-state.js");
|
||
const state = initRunState(cwd, [
|
||
{ name: "alpha", label: "alpha", fix: true },
|
||
{ name: "beta", label: "beta", fix: true },
|
||
]);
|
||
state.recon = {
|
||
complete: true,
|
||
path: join(cwd, ".pygienium", "recon.json"),
|
||
finishedAt: Date.now(),
|
||
};
|
||
// Pre-create alpha's on-disk artifacts so its completed entry has artifacts.
|
||
const alphaDir = join(cwd, ".pygienium", "checks", "alpha");
|
||
await mkdir(alphaDir, { recursive: true });
|
||
await writeFile(
|
||
join(alphaDir, "findings.md"),
|
||
"# alpha findings\nalpha-scan\n",
|
||
);
|
||
await writeFile(
|
||
join(alphaDir, "changes.md"),
|
||
"# alpha changes\nalpha-fix\n",
|
||
);
|
||
markComplete(state, "alpha");
|
||
// beta was interrupted mid-analysis — left at in_progress.
|
||
applyPhaseStatus(state, "beta", PHASE_RECON, "complete");
|
||
applyPhaseStatus(state, "beta", PHASE_ANALYSIS, "in_progress");
|
||
await saveRunState(state);
|
||
|
||
// Resume via all-run: only beta should re-dispatch.
|
||
const out = await captureStdout(() =>
|
||
handleAllCommand("--fix", stubCtx(cwd)),
|
||
);
|
||
expect(track.dispatched).not.toContain("alpha");
|
||
expect(track.dispatched).toContain("beta");
|
||
expect(out.join("\n")).toContain("skipping");
|
||
|
||
const after = await loadRunState(cwd);
|
||
expect(after?.status).toBe("complete");
|
||
expect(after?.checks.alpha.status).toBe("complete");
|
||
expect(after?.checks.beta.status).toBe("complete");
|
||
});
|
||
|
||
it("runAllChecks supports scan-only (no fix phase, no changes artifacts)", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
const outcome = await runAllChecks({ cwd });
|
||
expect(outcome.ran).toEqual(["alpha"]);
|
||
expect(outcome.skipped).toEqual([]);
|
||
expect(outcome.status).toBe("complete");
|
||
const state = await loadRunState(cwd);
|
||
expect(state?.checks.alpha.fix).toBe(false);
|
||
expect(state?.checks.alpha.changes).toBeUndefined();
|
||
// Summary still written.
|
||
const summary = await readFile(outcome.summaryPath, "utf8");
|
||
expect(summary).toContain("## alpha — complete");
|
||
expect(outcome.summaryPath).toBe(allSummaryPath(cwd));
|
||
});
|
||
|
||
it("renders a summary even when no checks are registered/selected", async () => {
|
||
const outcome = await runAllChecks({ cwd, only: ["nonexistent"] });
|
||
expect(outcome.ran).toEqual([]);
|
||
const summary = await readFile(outcome.summaryPath, "utf8");
|
||
expect(summary).toContain("# Pygienium all-run summary");
|
||
expect(summary).toContain("- checks: 0");
|
||
});
|
||
|
||
it("renderAllSummary reflects per-check statuses and fix tags", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
await runAllChecks({ cwd, fix: true });
|
||
const state = (await loadRunState(cwd))!;
|
||
const md = renderAllSummary(state, selectChecks());
|
||
expect(md).toContain("## alpha — complete (--fix)");
|
||
expect(md).toContain("phases: recon:C");
|
||
});
|
||
|
||
it("all-summary never shows a stale error under a complete check", async () => {
|
||
const { initRunState } = await import("../src/run-state.js");
|
||
const state = initRunState(cwd, [
|
||
{ name: "alpha", label: "Alpha", fix: false },
|
||
]);
|
||
const alpha = fakeCheck("alpha");
|
||
// A hand-edited/legacy state can carry an error on a completed check
|
||
// (the MagnaFluo run showed exactly this shape).
|
||
markCheckStatus(state, "alpha", "complete", "legacy verify error");
|
||
const md = renderAllSummary(state, [alpha]);
|
||
expect(md).toContain("## alpha — complete");
|
||
expect(md).not.toContain("- error:");
|
||
// A genuinely failed check still surfaces its error.
|
||
markCheckStatus(state, "alpha", "failed", "scan exploded");
|
||
const md2 = renderAllSummary(state, [alpha]);
|
||
expect(md2).toContain("- error: scan exploded");
|
||
});
|
||
|
||
it("the unified strip surfaces every check name over the run", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
const out = await captureStdout(() => handleAllCommand("", stubCtx(cwd)));
|
||
const joined = out.join("\n");
|
||
// Each check name appears in the strip line at least once.
|
||
expect(joined).toContain("Alpha");
|
||
expect(joined).toContain("Beta");
|
||
expect(joined).toContain("all [");
|
||
});
|
||
|
||
it("renders only the unified footer widget in UI mode (no per-check footer)", async () => {
|
||
registerCheck(fakeCheck("alpha"));
|
||
registerCheck(fakeCheck("beta"));
|
||
const calls: Array<[string, string[] | undefined]> = [];
|
||
const ui = {
|
||
setWidget: (key: string, content: string[] | undefined) => {
|
||
calls.push([key, content]);
|
||
},
|
||
} as never; // stub ExtensionUIContext (tests have no pi type imports)
|
||
|
||
await runAllChecks({ cwd, ui, hasUI: true });
|
||
|
||
const keys = new Set(calls.map(([k]) => k));
|
||
// The all-run footer drives the belowEditor widget area under its own key…
|
||
expect(keys.has("pygienium-all")).toBe(true);
|
||
// …and per-check footers never claim their slot (footer:false), so two
|
||
// overviews never compete over the same widget area.
|
||
expect(keys.has("pygienium")).toBe(false);
|
||
// The widget is cleared when the run completes.
|
||
const last = calls[calls.length - 1];
|
||
expect(last).toBeDefined();
|
||
expect(last?.[0]).toBe("pygienium-all");
|
||
expect(last?.[1]).toBeUndefined();
|
||
});
|
||
});
|