initial import: @mikefreno/omp-pygenium (omp port)
This commit is contained in:
235
tests/per-check-resume.test.ts
Normal file
235
tests/per-check-resume.test.ts
Normal file
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* per-check-resume.test.ts — `/pygienium-<check>` resume semantics.
|
||||
*
|
||||
* The per-check command must be resume-aware (parity with
|
||||
* `/pygienium-all` and `/pygienium-resume`): a check already terminal
|
||||
* (`complete`/`skipped`) is skipped unless `--fresh`, and a failed check is
|
||||
* re-dispatched. This is what lets "running the original command again"
|
||||
* recover a partial run instead of blindly re-running every phase.
|
||||
*
|
||||
* A stateful tracker wraps the fake runner: the first call produces no
|
||||
* artifact (verify fails), the second writes findings.md (verify passes) —
|
||||
* modelling the intermittent empty-output bug recovering on retry.
|
||||
*/
|
||||
import { describe, expect, it, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdtemp, mkdir, rm, writeFile, readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
clearChecks,
|
||||
registerCheck,
|
||||
type CheckDefinition,
|
||||
} from "../src/checks/registry.js";
|
||||
import {
|
||||
setAgentRunner,
|
||||
resetAgentRunner,
|
||||
fakeAgentRunner,
|
||||
type AgentRunner,
|
||||
} from "../src/agent-runner.js";
|
||||
import { handleCheckCommand, type PygieniumCtx } from "../src/commands.js";
|
||||
import { loadRunState } from "../src/run-state.js";
|
||||
|
||||
function stubCtx(cwd: string): PygieniumCtx {
|
||||
return { cwd, hasUI: false, ui: undefined } as PygieniumCtx;
|
||||
}
|
||||
|
||||
/** Capture process.stdout.write lines for the duration of `fn`. */
|
||||
async function captureStdout(fn: () => Promise<void>): Promise<string[]> {
|
||||
const out: string[] = [];
|
||||
const write = process.stdout.write.bind(process.stdout);
|
||||
(process.stdout as { write: (chunk: unknown) => boolean }).write = (
|
||||
chunk: unknown,
|
||||
) => {
|
||||
out.push(String(chunk).replace(/\r?\n$/, ""));
|
||||
return true;
|
||||
};
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
(process.stdout as { write: (chunk: unknown) => boolean }).write = write;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Check whose fake runner writes findings.md and (with --fix) changes.md. */
|
||||
function fakeCheck(name: string): CheckDefinition {
|
||||
return {
|
||||
name,
|
||||
label: name,
|
||||
description: `${name} check`,
|
||||
agentName: "scanner",
|
||||
fixAgentName: "fixer",
|
||||
phaseId: "scan",
|
||||
buildScanTask: (_cwd, scope) =>
|
||||
`!write .pygienium/checks/${name}/findings.md # ${name} findings\nscan-target:${scope.target}\n!echo ${name}-scan`,
|
||||
buildFixTask: (_cwd, _scope, findings) =>
|
||||
`!write .pygienium/checks/${name}/changes.md # ${name} changes\nbased-on:${findings.split("\n")[0] ?? ""}\n!echo ${name}-fix`,
|
||||
// Verify hook asserting findings.md exists — like the real checks.
|
||||
verify: async (scope) => {
|
||||
const { stat } = await import("node:fs/promises");
|
||||
const f = join(scope.cwd, ".pygienium", "checks", name, "findings.md");
|
||||
try {
|
||||
await stat(f);
|
||||
} catch {
|
||||
return `${name} verify: expected findings.md at ${f} after scan, none found.`;
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
gate: () => undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** Tracker: records dispatched agent tasks then delegates to the fake runner. */
|
||||
function trackingRunner(): { runner: AgentRunner; dispatched: string[] } {
|
||||
const dispatched: string[] = [];
|
||||
const runner: AgentRunner = async (opts) => {
|
||||
const m = /pygienium\/checks\/([^/]+)\//.exec(opts.task);
|
||||
if (m) dispatched.push(m[1] as string);
|
||||
return fakeAgentRunner(opts);
|
||||
};
|
||||
return { runner, dispatched };
|
||||
}
|
||||
|
||||
/**
|
||||
* Stateful runner: the first call returns ok with no artifact (verify fails),
|
||||
* the second writes findings.md (verify passes). Models the empty-output bug
|
||||
* recovering on retry.
|
||||
*/
|
||||
function flakyThenOkRunner(name: string): {
|
||||
runner: AgentRunner;
|
||||
calls: number;
|
||||
} {
|
||||
let calls = 0;
|
||||
const runner: AgentRunner = async (opts) => {
|
||||
calls++;
|
||||
if (calls === 1) {
|
||||
return { ok: true, text: "" };
|
||||
}
|
||||
return fakeAgentRunner(opts);
|
||||
};
|
||||
return { runner, calls: 0 };
|
||||
}
|
||||
|
||||
describe("/pygienium-<check> resume semantics", () => {
|
||||
let cwd: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
clearChecks();
|
||||
cwd = await mkdtemp(join(tmpdir(), "pygienium-pcr-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetAgentRunner();
|
||||
await rm(cwd, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
it("skips an already-complete check and does not re-dispatch the agent", async () => {
|
||||
const track = trackingRunner();
|
||||
setAgentRunner(track.runner);
|
||||
const check = fakeCheck("alpha");
|
||||
registerCheck(check);
|
||||
|
||||
// First run: completes and writes findings.md.
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
expect(track.dispatched).toEqual(["alpha"]);
|
||||
const state1 = await loadRunState(cwd);
|
||||
expect(state1?.checks.alpha.status).toBe("complete");
|
||||
|
||||
// Second run: terminal → skipped, no agent dispatch.
|
||||
const out = await captureStdout(() =>
|
||||
handleCheckCommand(check, "", stubCtx(cwd)),
|
||||
);
|
||||
expect(track.dispatched).toEqual(["alpha"]); // unchanged
|
||||
expect(out.join("\n")).toContain("already complete");
|
||||
expect(out.join("\n")).toContain("--fresh");
|
||||
const state2 = await loadRunState(cwd);
|
||||
expect(state2?.checks.alpha.status).toBe("complete");
|
||||
});
|
||||
|
||||
it("--fresh re-runs a completed check from scratch", async () => {
|
||||
const track = trackingRunner();
|
||||
setAgentRunner(track.runner);
|
||||
const check = fakeCheck("beta");
|
||||
registerCheck(check);
|
||||
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
expect(track.dispatched).toEqual(["beta"]);
|
||||
|
||||
await captureStdout(() =>
|
||||
handleCheckCommand(check, "--fresh", stubCtx(cwd)),
|
||||
);
|
||||
// Dispatched again (now twice total).
|
||||
expect(track.dispatched).toEqual(["beta", "beta"]);
|
||||
const state = await loadRunState(cwd);
|
||||
expect(state?.checks.beta.status).toBe("complete");
|
||||
});
|
||||
|
||||
it("re-runs a failed check and recovers when the agent produces the artifact on retry", async () => {
|
||||
const flaky = flakyThenOkRunner("gamma");
|
||||
// Expose the live call count via closure read after the run.
|
||||
let calls = 0;
|
||||
const runner: AgentRunner = async (opts) => {
|
||||
calls++;
|
||||
if (calls === 1) {
|
||||
return { ok: true, text: "" };
|
||||
}
|
||||
return fakeAgentRunner(opts);
|
||||
};
|
||||
void flaky; // (flakyThenOkRunner kept as a reference shape; use `runner` below)
|
||||
setAgentRunner(runner);
|
||||
|
||||
const check = fakeCheck("gamma");
|
||||
registerCheck(check);
|
||||
|
||||
// First run: agent returns ok with no artifact → verify fails.
|
||||
await handleCheckCommand(check, "", stubCtx(cwd));
|
||||
const state1 = await loadRunState(cwd);
|
||||
expect(state1?.checks.gamma.status).toBe("failed");
|
||||
expect(state1?.checks.gamma.error).toContain("verify");
|
||||
expect(state1?.checks.gamma.error).toContain("findings.md");
|
||||
|
||||
// Resume: re-dispatch the failed check. Agent writes findings.md this
|
||||
// time → verify passes → complete.
|
||||
await captureStdout(() => handleCheckCommand(check, "", stubCtx(cwd)));
|
||||
const state2 = await loadRunState(cwd);
|
||||
expect(state2?.checks.gamma.status).toBe("complete");
|
||||
expect(state2?.checks.gamma.error).toBeUndefined();
|
||||
// The verify phase is now complete (not failed).
|
||||
const verify = state2?.checks.gamma.phases.find((p) => p.id === "verify");
|
||||
expect(verify?.status).toBe("complete");
|
||||
// And the artifact exists on disk.
|
||||
const findings = await readFile(
|
||||
join(cwd, ".pygienium", "checks", "gamma", "findings.md"),
|
||||
"utf8",
|
||||
);
|
||||
expect(findings).toContain("gamma findings");
|
||||
});
|
||||
|
||||
it("does not treat a failed check as terminal (resume re-dispatches it)", async () => {
|
||||
const track = trackingRunner();
|
||||
setAgentRunner(track.runner);
|
||||
const check = fakeCheck("delta");
|
||||
// Override verify to always fail so the check lands in `failed`.
|
||||
const alwaysFailing: CheckDefinition = {
|
||||
...check,
|
||||
name: "delta",
|
||||
verify: async () => "delta verify: forced failure",
|
||||
};
|
||||
registerCheck(alwaysFailing);
|
||||
|
||||
await handleCheckCommand(alwaysFailing, "", stubCtx(cwd));
|
||||
expect(track.dispatched).toEqual(["delta"]);
|
||||
const state1 = await loadRunState(cwd);
|
||||
expect(state1?.checks.delta.status).toBe("failed");
|
||||
|
||||
// Re-running the command re-dispatches (failed is NOT terminal).
|
||||
await captureStdout(() =>
|
||||
handleCheckCommand(alwaysFailing, "", stubCtx(cwd)),
|
||||
);
|
||||
expect(track.dispatched).toEqual(["delta", "delta"]);
|
||||
const state2 = await loadRunState(cwd);
|
||||
expect(state2?.checks.delta.status).toBe("failed"); // still failing
|
||||
});
|
||||
});
|
||||
|
||||
void mkdir; // silence unused-import lint under some configs
|
||||
Reference in New Issue
Block a user