Initial commit: pygenium as git submodule
This commit is contained in:
354
tests/all.test.ts
Normal file
354
tests/all.test.ts
Normal file
@@ -0,0 +1,354 @@
|
||||
/**
|
||||
* all.test.ts — integration test for the `/pygienium-all` orchestrator (task 12).
|
||||
*
|
||||
* Mirrors the spec scenario: run `/pygienium-all` on a small repo and assert:
|
||||
* - every registered check runs exactly once in registry order;
|
||||
* - run-state shows all checks complete and the overall run complete;
|
||||
* - `pygienium/all-summary.md` is present and lists per-check outcomes;
|
||||
* - `--only=alpha,gamma` narrows the candidate set preserving order;
|
||||
* - interrupted/resumed runs re-dispatch non-terminal checks while skipping
|
||||
* terminal ones, unless `--fresh` resets everything.
|
||||
*
|
||||
* A tracker wraps the fake agent runner so we can assert dispatch order and
|
||||
* counts without a model.
|
||||
*/
|
||||
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdtemp, mkdir, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
clearChecks,
|
||||
registerCheck,
|
||||
type CheckDefinition,
|
||||
} from "../src/checks/registry.js";
|
||||
import {
|
||||
setAgentRunner,
|
||||
resetAgentRunner,
|
||||
fakeAgentRunner,
|
||||
type AgentRunner,
|
||||
} from "../src/agent-runner.js";
|
||||
import { handleAllCommand, type PygieniumCtx } from "../src/commands.js";
|
||||
import {
|
||||
parseAllArgs,
|
||||
runAllChecks,
|
||||
allSummaryPath,
|
||||
renderAllSummary,
|
||||
selectChecks,
|
||||
} from "../src/modes/all.js";
|
||||
import {
|
||||
loadRunState,
|
||||
markCheckStatus,
|
||||
applyPhaseStatus,
|
||||
PHASE_RECON,
|
||||
PHASE_ANALYSIS,
|
||||
PHASE_FIX,
|
||||
PHASE_VERIFY,
|
||||
PHASE_CLEANUP,
|
||||
} from "../src/run-state.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
|
||||
/** Build a deterministic check whose fake runner writes on-disk artifacts. */
|
||||
function fakeCheck(name: string): CheckDefinition {
|
||||
return {
|
||||
name,
|
||||
label: name.charAt(0).toUpperCase() + name.slice(1),
|
||||
description: `${name} check`,
|
||||
agentName: "scanner",
|
||||
fixAgentName: "fixer",
|
||||
phaseId: "scan",
|
||||
buildScanTask: (_cwd, scope) =>
|
||||
`!write pygienium/checks/${name}/findings.md # ${name} findings\nscan-target:${scope.target}\n!echo ${name}-scan`,
|
||||
buildFixTask: (_cwd, _scope, findings) =>
|
||||
`!write pygienium/checks/${name}/changes.md # ${name} changes\nbased-on:${findings.split("\n")[0] ?? ""}\n!echo ${name}-fix`,
|
||||
gate: () => undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function stubCtx(cwd: string): PygieniumCtx {
|
||||
return { cwd, mode: "print", hasUI: false, ui: undefined } as PygieniumCtx;
|
||||
}
|
||||
|
||||
/** Tracker: records dispatched agent tasks then delegates to the fake runner. */
|
||||
function trackingRunner(): { runner: AgentRunner; dispatched: string[] } {
|
||||
const dispatched: string[] = [];
|
||||
const runner: AgentRunner = async (opts) => {
|
||||
const m = /pygienium\/checks\/([^/]+)\//.exec(opts.task);
|
||||
if (m) dispatched.push(m[1] as string);
|
||||
return fakeAgentRunner(opts);
|
||||
};
|
||||
return { runner, dispatched };
|
||||
}
|
||||
|
||||
/** Capture process.stdout.write lines for the duration of `fn`. */
|
||||
async function captureStdout(fn: () => Promise<void>): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
const write = process.stdout.write.bind(process.stdout);
|
||||
(process.stdout as { write: (chunk: unknown) => boolean }).write = (
|
||||
chunk: unknown,
|
||||
) => {
|
||||
out.push(String(chunk).replace(/\r?\n$/, ""));
|
||||
return true;
|
||||
};
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
(process.stdout as { write: (chunk: unknown) => boolean }).write = write;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Mark a check as fully complete in the run-state (helper for seeding). */
|
||||
function markComplete(
|
||||
state: Parameters<typeof markCheckStatus>[0],
|
||||
name: string,
|
||||
): void {
|
||||
for (const phaseId of [
|
||||
PHASE_RECON,
|
||||
PHASE_ANALYSIS,
|
||||
PHASE_FIX,
|
||||
PHASE_VERIFY,
|
||||
PHASE_CLEANUP,
|
||||
]) {
|
||||
applyPhaseStatus(state, name, phaseId, "complete");
|
||||
}
|
||||
markCheckStatus(state, name, "complete");
|
||||
}
|
||||
|
||||
describe("/pygienium-all orchestrator (task 12)", () => {
|
||||
let cwd: string;
|
||||
let track: ReturnType<typeof trackingRunner>;
|
||||
|
||||
beforeEach(async () => {
|
||||
clearChecks();
|
||||
track = trackingRunner();
|
||||
setAgentRunner(track.runner);
|
||||
cwd = await mkdtemp(join(tmpdir(), "pygienium-all-"));
|
||||
// Seed a source file so the gate passes and recon has something to scan.
|
||||
await mkdir(join(cwd, "src"), { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetAgentRunner();
|
||||
await rm(cwd, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("parseAllArgs parses path, --fix, --fresh, and --only", () => {
|
||||
const p = parseAllArgs("subdir --fix --only=alpha,beta --fresh", cwd);
|
||||
expect(p.target).toBe(join(cwd, "subdir"));
|
||||
expect(p.fix).toBe(true);
|
||||
expect(p.fresh).toBe(true);
|
||||
expect(p.only).toEqual(["alpha", "beta"]);
|
||||
});
|
||||
|
||||
it("selectChecks preserves registry order for the --only subset", () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
registerCheck(fakeCheck("gamma"));
|
||||
const subset = selectChecks(["gamma", "alpha"]); // order in --only is irrelevant
|
||||
expect(subset.map((c) => c.name)).toEqual(["alpha", "gamma"]);
|
||||
expect(selectChecks().length).toBe(3);
|
||||
expect(selectChecks([]).length).toBe(3);
|
||||
});
|
||||
|
||||
it("runs every registered check exactly once in registry order", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
registerCheck(fakeCheck("gamma"));
|
||||
|
||||
await captureStdout(() => handleAllCommand("--fix", stubCtx(cwd)));
|
||||
|
||||
// Each check dispatched once for scan + once for fix (3 checks × 2 phases).
|
||||
const scanDispatches = track.dispatched.filter((n) => n !== undefined);
|
||||
expect(scanDispatches).toEqual([
|
||||
"alpha",
|
||||
"alpha",
|
||||
"beta",
|
||||
"beta",
|
||||
"gamma",
|
||||
"gamma",
|
||||
]);
|
||||
|
||||
const state = await loadRunState(cwd);
|
||||
expect(state?.status).toBe("complete");
|
||||
expect(state?.checks.alpha.status).toBe("complete");
|
||||
expect(state?.checks.beta.status).toBe("complete");
|
||||
expect(state?.checks.gamma.status).toBe("complete");
|
||||
expect(state?.recon.complete).toBe(true);
|
||||
});
|
||||
|
||||
it("writes pygienium/all-summary.md listing per-check outcomes", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
|
||||
await captureStdout(() => handleAllCommand("--fix", stubCtx(cwd)));
|
||||
|
||||
const summary = await readFile(allSummaryPath(cwd), "utf8");
|
||||
expect(summary).toContain("# Pygienium all-run summary");
|
||||
expect(summary).toContain("- status: complete");
|
||||
expect(summary).toContain("## alpha — complete");
|
||||
expect(summary).toContain("## beta — complete");
|
||||
expect(summary).toContain("## alpha — complete (--fix)");
|
||||
// Artifact paths + line counts are referenced.
|
||||
expect(summary).toContain("findings:");
|
||||
expect(summary).toContain("changes:");
|
||||
// Artifacts actually exist on disk.
|
||||
const alphaFindings = await readFile(
|
||||
join(cwd, "pygienium", "checks", "alpha", "findings.md"),
|
||||
"utf8",
|
||||
);
|
||||
expect(alphaFindings).toContain("alpha findings");
|
||||
});
|
||||
|
||||
it("--only narrows the run to the named subset", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
registerCheck(fakeCheck("gamma"));
|
||||
|
||||
await captureStdout(() =>
|
||||
handleAllCommand("--only=alpha,gamma", stubCtx(cwd)),
|
||||
);
|
||||
|
||||
// Only alpha + gamma dispatched (beta never touched).
|
||||
expect(track.dispatched).toContain("alpha");
|
||||
expect(track.dispatched).toContain("gamma");
|
||||
expect(track.dispatched).not.toContain("beta");
|
||||
|
||||
const state = await loadRunState(cwd);
|
||||
expect(state?.checks.alpha.status).toBe("complete");
|
||||
expect(state?.checks.gamma.status).toBe("complete");
|
||||
// beta was not part of the selected set, so has no entry.
|
||||
expect(state?.checks.beta).toBeUndefined();
|
||||
|
||||
const summary = await readFile(allSummaryPath(cwd), "utf8");
|
||||
expect(summary).toContain("## alpha");
|
||||
expect(summary).toContain("## gamma");
|
||||
expect(summary).not.toContain("## beta");
|
||||
});
|
||||
|
||||
it("skips terminal checks on re-run; --fresh re-runs them", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
|
||||
// First run: both complete.
|
||||
await captureStdout(() => handleAllCommand("--fix", stubCtx(cwd)));
|
||||
expect(track.dispatched.length).toBe(4); // 2 checks × 2 phases
|
||||
const firstAlphaFix = await readFile(
|
||||
join(cwd, "pygienium", "checks", "alpha", "changes.md"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
// Second run without --fresh: both already terminal → skipped.
|
||||
track.dispatched.length = 0;
|
||||
const out = await captureStdout(() =>
|
||||
handleAllCommand("--fix", stubCtx(cwd)),
|
||||
);
|
||||
expect(track.dispatched).toHaveLength(0);
|
||||
expect(out.join("\n")).toContain("skipping");
|
||||
const state2 = await loadRunState(cwd);
|
||||
expect(state2?.status).toBe("complete");
|
||||
|
||||
// Third run with --fresh: both re-dispatched from scratch.
|
||||
track.dispatched.length = 0;
|
||||
await captureStdout(() => handleAllCommand("--fix --fresh", stubCtx(cwd)));
|
||||
expect(track.dispatched).toEqual(["alpha", "alpha", "beta", "beta"]);
|
||||
const state3 = await loadRunState(cwd);
|
||||
expect(state3?.status).toBe("complete");
|
||||
// The fresh re-run overwrote alpha's changes.md (still valid content).
|
||||
const alphaFix2 = await readFile(
|
||||
join(cwd, "pygienium", "checks", "alpha", "changes.md"),
|
||||
"utf8",
|
||||
);
|
||||
expect(alphaFix2).toContain("alpha changes");
|
||||
void firstAlphaFix;
|
||||
});
|
||||
|
||||
it("resumes an interrupted run (re-dispatches non-terminal checks only)", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
|
||||
// Seed an interrupted run: alpha complete, beta pending (interrupted).
|
||||
const { initRunState, saveRunState } = await import("../src/run-state.js");
|
||||
const state = initRunState(cwd, [
|
||||
{ name: "alpha", label: "alpha", fix: true },
|
||||
{ name: "beta", label: "beta", fix: true },
|
||||
]);
|
||||
state.recon = {
|
||||
complete: true,
|
||||
path: join(cwd, ".pygienium", "recon.json"),
|
||||
finishedAt: Date.now(),
|
||||
};
|
||||
// Pre-create alpha's on-disk artifacts so its completed entry has artifacts.
|
||||
const alphaDir = join(cwd, "pygienium", "checks", "alpha");
|
||||
await mkdir(alphaDir, { recursive: true });
|
||||
await writeFile(
|
||||
join(alphaDir, "findings.md"),
|
||||
"# alpha findings\nalpha-scan\n",
|
||||
);
|
||||
await writeFile(
|
||||
join(alphaDir, "changes.md"),
|
||||
"# alpha changes\nalpha-fix\n",
|
||||
);
|
||||
markComplete(state, "alpha");
|
||||
// beta was interrupted mid-analysis — left at in_progress.
|
||||
applyPhaseStatus(state, "beta", PHASE_RECON, "complete");
|
||||
applyPhaseStatus(state, "beta", PHASE_ANALYSIS, "in_progress");
|
||||
await saveRunState(state);
|
||||
|
||||
// Resume via all-run: only beta should re-dispatch.
|
||||
const out = await captureStdout(() =>
|
||||
handleAllCommand("--fix", stubCtx(cwd)),
|
||||
);
|
||||
expect(track.dispatched).not.toContain("alpha");
|
||||
expect(track.dispatched).toContain("beta");
|
||||
expect(out.join("\n")).toContain("skipping");
|
||||
|
||||
const after = await loadRunState(cwd);
|
||||
expect(after?.status).toBe("complete");
|
||||
expect(after?.checks.alpha.status).toBe("complete");
|
||||
expect(after?.checks.beta.status).toBe("complete");
|
||||
});
|
||||
|
||||
it("runAllChecks supports scan-only (no fix phase, no changes artifacts)", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
const outcome = await runAllChecks({ cwd });
|
||||
expect(outcome.ran).toEqual(["alpha"]);
|
||||
expect(outcome.skipped).toEqual([]);
|
||||
expect(outcome.status).toBe("complete");
|
||||
const state = await loadRunState(cwd);
|
||||
expect(state?.checks.alpha.fix).toBe(false);
|
||||
expect(state?.checks.alpha.changes).toBeUndefined();
|
||||
// Summary still written.
|
||||
const summary = await readFile(outcome.summaryPath, "utf8");
|
||||
expect(summary).toContain("## alpha — complete");
|
||||
expect(outcome.summaryPath).toBe(allSummaryPath(cwd));
|
||||
});
|
||||
|
||||
it("renders a summary even when no checks are registered/selected", async () => {
|
||||
const outcome = await runAllChecks({ cwd, only: ["nonexistent"] });
|
||||
expect(outcome.ran).toEqual([]);
|
||||
const summary = await readFile(outcome.summaryPath, "utf8");
|
||||
expect(summary).toContain("# Pygienium all-run summary");
|
||||
expect(summary).toContain("- checks: 0");
|
||||
});
|
||||
|
||||
it("renderAllSummary reflects per-check statuses and fix tags", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
await runAllChecks({ cwd, fix: true });
|
||||
const state = (await loadRunState(cwd))!;
|
||||
const md = renderAllSummary(state, selectChecks());
|
||||
expect(md).toContain("## alpha — complete (--fix)");
|
||||
expect(md).toContain("phases: recon:C");
|
||||
});
|
||||
|
||||
it("the unified strip surfaces every check name over the run", async () => {
|
||||
registerCheck(fakeCheck("alpha"));
|
||||
registerCheck(fakeCheck("beta"));
|
||||
const out = await captureStdout(() => handleAllCommand("", stubCtx(cwd)));
|
||||
const joined = out.join("\n");
|
||||
// Each check name appears in the strip line at least once.
|
||||
expect(joined).toContain("Alpha");
|
||||
expect(joined).toContain("Beta");
|
||||
expect(joined).toContain("all [");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user