pygienium/src/checks/todos.ts

/**
 * checks/todos.ts — "TODOs & stubs" check.
 *
 * Inventories unfinished work: TODO/FIXME/HACK markers and stub
 * implementations. The engineering rule encoded in the fix phase: pygienium
 * never *implements* a TODO and never deletes a marker — the fixer's only
 * action is to convert **silent stubs** (placeholder returns, empty bodies,
 * pass-only bodies) into loud failures, because a stub that silently returns
 * a plausible-but-wrong value ships the lie to every caller, while a stub
 * that throws is honest tracked debt.
 *
 * Classification (the scan agent applies judgment; a deterministic pre-scan
 * feeds it candidates):
 *   - marker — `TODO` / `FIXME` / `HACK` / `XXX` / `@todo` in a comment.
 *   - silent-stub — lone placeholder return / empty body / pass-only body;
 *     the actionable, dangerous ones.
 *   - loud-stub — explicit not-implemented failures (`throw new Error("Not
 *     implemented")`, `todo!()`, `raise NotImplementedError`, `TODO("...")`);
 *     already failing loudly → tracked debt, fixer never touches them.
 *   - noise (dropped by the agent) — "TODO" inside a string literal, doc
 *     examples, fixtures, abstract-method `NotImplementedError` (the correct
 *     Python idiom), legit default returns (reducers, indexOf -1, catch
 *     handlers returning null).
 *
 * Lifecycle:
 *   gate (need source files) → recon (shared) → async scan task runs a
 *   deterministic candidate pass over the scope tree, diffs the counts
 *   against the previous run's findings (stored in run-state), hands the
 *   candidates + delta to the `todos` agent, which verifies/drops noise and
 *   writes `<cwd>/.pygienium/checks/todos/findings.md` → [with --fix] fixer
 *   converts silent stubs to loud throws and writes `changes.md`.
 *
 * Registering this file is the ONLY wiring needed: `index.ts` auto-discovers
 * `src/checks/*.ts`, so dropping this file exposes `/pygienium-todos`.
 *
 * @module pygienium/checks/todos
 */

import { readFile, readdir, stat } from "node:fs/promises";
import { join, relative } from "node:path";
import { loadRunState } from "../run-state.js";
import {
	registerCheck,
	type CheckDefinition,
	type CheckScope,
} from "./registry.js";
import {
	hasScopeSources,
	isScopeSource,
	SCOPE_EXCLUDE_DIRS,
	scopeRulesMarkdown,
} from "./scope.js";

/** Output directory for this check's persistent reports. */
export function todosOutputDir(cwd: string): string {
	return join(cwd, ".pygienium", "checks", "todos");
}

/** `findings.md` path for this check. */
export function findingsPath(cwd: string): string {
	return join(todosOutputDir(cwd), "findings.md");
}

/** `changes.md` path for this check. */
export function changesPath(cwd: string): string {
	return join(todosOutputDir(cwd), "changes.md");
}

export type TodoKind = "marker" | "silent-stub" | "loud-stub";

/** One candidate line the deterministic pre-scan flagged. */
export interface TodoCandidate {
	/** Absolute path of the file. */
	path: string;
	/** 1-based line number. */
	line: number;
	kind: TodoKind;
	/** Matched token (e.g. `TODO`, `Not implemented`, `empty-body`). */
	snippet: string;
	/** The trimmed line content. */
	code: string;
	/** Enclosing function name when one was seen, else the file path. */
	context: string;
}

/**
 * Marker tokens: an unfinished-work note in a comment. Case-insensitive;
 * `@todo\b` (not `@todos`) and `\bHACK\b` (not `hacking`).
 */
const MARKER_RE = /\b(?:TODO|FIXME|HACK)\b|\bXXX\b|@todo\b/i;

/**
 * Loud-stub tokens: explicit not-implemented failures. `not implemented`
 * covers `throw new Error("Not implemented")` and `panic!("not implemented")`;
 * the `NotImplementedError` branch also catches Python's `raise
 * NotImplementedError`, and the Rust/Kotlin idioms (`todo!()`, `TODO("...")`)
 * are matched explicitly.
 */
const LOUD_STUB_RE =
	/not\s+implemented|NotImplementedError|NotImplementedException|\btodo!\s*\(|unimplemented!\s*\(|\bTODO\s*\(/i;

/** A lone placeholder return (`return 0;` / `return "";` / `return null;` …). */
const PLACEHOLDER_RETURN_RE =
	/^\s*return\s+(?:null|undefined|0(?:\.0)?|""|''|\[\]|\{\}|None|nil|false)\s*;?\s*(?:\/\/.*)?$/;

/** Function/arrow header lines worth inspecting for a stub body. */
const FN_HEADER_RE =
	/\b(?:function|def|func|fun|fn)\s+\w+|(?:const|let|var)\s+\w+\s*=\s*(?:async\s*)?\s*(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/;

/** Single-line placeholder body: `function x() { return 0; }`. */
const SINGLE_PLACEHOLDER_BODY_RE =
	/\{\s*return\s+(?:null|undefined|0(?:\.0)?|""|''|\[\]|\{\}|None|nil|false)\s*;?\s*\}/;

/** Single-line arrow expression body: `const f = () => 0;`. */
const ARROW_PLACEHOLDER_RE =
	/=>\s*(?:null|undefined|0(?:\.0)?|""|''|\[\]|\{\}|false)\s*;?\s*$/;

/** Empty single-line body: `function notify(): void {}`. */
const EMPTY_BODY_RE = /\{\s*\}/;

/** Hard cap on candidates so a huge tree can't blow the task prompt. */
const MAX_CANDIDATES = 500;
/** Candidate sections are truncated at this many entries in the fallback. */
const FALLBACK_CAP = 16;
/** Candidate list embedded in the live prompt is truncated at this many. */
const PROMPT_CAP = 40;
/**
 * Max characters of a candidate's code line embedded in the task prompt.
 * Generated/bundled single lines can be hundreds of KB (e.g. minified assets
 * sneaking past scope); embedding them wholesale balloons the task to
 * megabytes and chokes the sub-agent. `path:line` plus a truncated prefix is
 * enough to classify — the agent can read the file for full context.
 */
const CODE_DISPLAY_CAP = 160;

/** Truncate a candidate's code line for prompt embedding. */
function displayCode(code: string): string {
	return code.length > CODE_DISPLAY_CAP
		? `${code.slice(0, CODE_DISPLAY_CAP)}…`
		: code;
}

/** Extract the declared function name from a header line, when present. */
function headerName(line: string): string | undefined {
	const decl =
		/(?:function|def|func|fun|fn|class)\s+([A-Za-z_$][\w$]*)/.exec(line) ??
		/(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?\s*(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/.exec(
			line,
		);
	return decl?.[1];
}

/** Index of the next non-blank line at or after `start`, else `undefined`. */
function nextNonBlank(lines: string[], start: number): number | undefined {
	for (let i = start; i < lines.length; i++) {
		if ((lines[i] as string).trim()) return i;
	}
	return undefined;
}

/**
 * Walk the target collecting implementation-source files, honouring
 * {@link SCOPE_EXCLUDE_DIRS} and {@link isScopeSource} (same rules as
 * dead-code's walker).
 */
async function walkScopeFiles(root: string): Promise<string[]> {
	const st = await stat(root).catch(() => undefined);
	if (!st) return [];
	if (st.isFile()) return isScopeSource(root) ? [root] : [];
	const out: string[] = [];
	const stack = [root];
	while (stack.length > 0) {
		const dir = stack.pop() as string;
		let entries;
		try {
			entries = await readdir(dir, { withFileTypes: true });
		} catch {
			continue;
		}
		for (const entry of entries) {
			const full = join(dir, entry.name);
			if (entry.isDirectory()) {
				if (SCOPE_EXCLUDE_DIRS.has(entry.name)) continue;
				stack.push(full);
			} else if (entry.isFile() && isScopeSource(entry.name)) {
				out.push(full);
			}
		}
	}
	return out.sort();
}

/**
 * Deterministic pre-scan: flag marker/loud-stub/silent-stub candidates across
 * the target's scope tree. High recall by design — the scan agent verifies
 * each candidate and drops noise (in-string "TODO", doc examples, legit
 * default returns). Pure function of the tree: unit-testable without agents.
 */
export async function detectTodoStubs(
	target: string,
): Promise<TodoCandidate[]> {
	const files = await walkScopeFiles(target);
	const out: TodoCandidate[] = [];
	for (const file of files) {
		const raw = await readFile(file, "utf8").catch(() => "");
		const lines = raw.split("\n");
		let lastFn = "";
		for (let i = 0; i < lines.length; i++) {
			const code = lines[i] as string;
			const trimmed = code.trim();
			if (!trimmed) continue;

			const loud = LOUD_STUB_RE.exec(trimmed);
			if (loud) {
				out.push({
					path: file,
					line: i + 1,
					kind: "loud-stub",
					snippet: loud[0].slice(0, 40),
					code: trimmed,
					context: lastFn,
				});
				continue;
			}
			const marker = MARKER_RE.exec(trimmed);
			if (marker) {
				out.push({
					path: file,
					line: i + 1,
					kind: "marker",
					snippet: marker[0],
					code: trimmed,
					context: lastFn,
				});
				continue;
			}

			if (FN_HEADER_RE.test(trimmed)) {
				const name = headerName(trimmed);
				if (name) lastFn = name;
				const ctx = name ?? lastFn;
				// Single-line stub forms.
				if (
					SINGLE_PLACEHOLDER_BODY_RE.test(trimmed) ||
					ARROW_PLACEHOLDER_RE.test(trimmed)
				) {
					out.push({
						path: file,
						line: i + 1,
						kind: "silent-stub",
						snippet: "placeholder-return",
						code: trimmed,
						context: ctx,
					});
					continue;
				}
				if (
					EMPTY_BODY_RE.test(trimmed) &&
					!/\b(?:return|throw)\b/.test(trimmed)
				) {
					out.push({
						path: file,
						line: i + 1,
						kind: "silent-stub",
						snippet: "empty-body",
						code: trimmed,
						context: ctx,
					});
					continue;
				}
				// Multi-line forms: inspect the first non-blank body line.
				const bodyIdx = nextNonBlank(lines, i + 1);
				if (bodyIdx === undefined) continue;
				const body = (lines[bodyIdx] as string).trim();
				if (body === "}") {
					out.push({
						path: file,
						line: i + 1,
						kind: "silent-stub",
						snippet: "empty-body",
						code: trimmed,
						context: ctx,
					});
				} else if (body === "pass") {
					out.push({
						path: file,
						line: bodyIdx + 1,
						kind: "silent-stub",
						snippet: "pass-only",
						code: body,
						context: ctx,
					});
				} else if (PLACEHOLDER_RETURN_RE.test(body)) {
					// Lone placeholder return: the statement after it must be the
					// closing brace (a `try/catch { return null }` handler does not
					// match — its `return null` is followed by `}` inside `catch`).
					const after = nextNonBlank(lines, bodyIdx + 1);
					if (after !== undefined && (lines[after] as string).trim() === "}") {
						out.push({
							path: file,
							line: bodyIdx + 1,
							kind: "silent-stub",
							snippet: "placeholder-return",
							code: body,
							context: ctx,
						});
					}
				}
			}
		}
		if (out.length >= MAX_CANDIDATES) break;
	}
	return out;
}

/** Counts by kind across a candidate list. */
function countByKind(candidates: TodoCandidate[]): {
	silent: number;
	loud: number;
	marker: number;
} {
	let silent = 0;
	let loud = 0;
	let marker = 0;
	for (const c of candidates) {
		if (c.kind === "silent-stub") silent++;
		else if (c.kind === "loud-stub") loud++;
		else marker++;
	}
	return { silent, loud, marker };
}

/** Previous run's verified counts, parsed from run-state findings text. */
const PRIOR_SUMMARY_RE =
	/todos:\s*(\d+)\s+silent\s+stub\(s\)?,\s*(\d+)\s+loud\s+stub\(s\)?,\s*(\d+)\s+marker\(s\)?/;

/**
 * Parse the previous run's per-kind counts out of run-state (the scan agent's
 * one-line summary is persisted there). `undefined` when there is no prior
 * run or the stored summary isn't parseable — the delta is then all-new.
 */
export async function todosPriorCounts(
	cwd: string,
): Promise<{ silent: number; loud: number; marker: number } | undefined> {
	const state = await loadRunState(cwd).catch(() => undefined);
	const stored = state?.checks["todos"]?.findings;
	if (!stored) return undefined;
	const m = PRIOR_SUMMARY_RE.exec(stored);
	if (!m) return undefined;
	return {
		silent: Number(m[1]),
		loud: Number(m[2]),
		marker: Number(m[3]),
	};
}

/**
 * Render the deterministic report the fake runner writes (and the real agent
 * uses as a shape reference): one pipe-separated line, sections per kind.
 */
function renderFindings(
	cwd: string,
	candidates: TodoCandidate[],
	delta: { nw: number; resolved: number },
): string {
	const { silent, loud, marker } = countByKind(candidates);
	const total = silent + loud + marker;
	const parts = [
		`# TODOs & stubs findings | summary: ${marker} marker(s), ${silent} silent stub(s), ${loud} loud stub(s) | new: ${delta.nw} | resolved: ${delta.resolved} | reviewed: ${candidates.length}`,
	];
	if (total === 0) {
		parts.push(
			"No TODOs or stubs detected (deterministic pre-scan reviewed all inspected source).",
		);
		return parts.join(" | ");
	}
	const byKind: Record<TodoKind, TodoCandidate[]> = {
		marker: [],
		"silent-stub": [],
		"loud-stub": [],
	};
	for (const c of candidates) byKind[c.kind].push(c);
	const dump = (title: string, list: TodoCandidate[]): void => {
		parts.push(`## ${title}`);
		list.slice(0, FALLBACK_CAP).forEach((c, i) => {
			parts.push(
				`### ${i + 1}. ${relative(cwd, c.path)}:${c.line} — ${displayCode(c.code)} | snippet: ${c.snippet} | context: ${c.context || relative(cwd, c.path)}`,
			);
		});
		if (list.length > FALLBACK_CAP) {
			parts.push(`... and ${list.length - FALLBACK_CAP} more (truncated)`);
		}
	};
	dump("TODO markers", byKind.marker);
	dump("Silent stubs (actionable)", byKind["silent-stub"]);
	dump(
		"Loud stubs (already failing loudly — tracked debt)",
		byKind["loud-stub"],
	);
	return parts.join(" | ");
}

/**
 * Gate: skip when the cwd has no inspectable source files at all — a workspace
 * with zero source files gives the scanner nothing to analyse.
 */
function todosGate(cwd: string): string | undefined {
	try {
		if (hasScopeSources(cwd)) return undefined;
	} catch {
		// unreadable cwd → let the agent decide; don't block.
		return undefined;
	}
	return "no source files found to inspect";
}

/**
 * Build the scan task. Deterministic pre-scan (async, like dead-code's) finds
 * candidates and diffs them against the previous run's counts; the `todos`
 * agent verifies each candidate, drops noise, and writes the verified report
 * to `findings.md`. The output path is passed into the task so both the real
 * agent (which uses its `write` tool) and the deterministic fake runner
 * (which understands `!write <path> <text>`) persist to the same location.
 */
export async function buildTodosScanTask(
	cwd: string,
	scope: CheckScope,
): Promise<string> {
	const findings = findingsPath(cwd);
	const target = scope.target;
	const candidates = await detectTodoStubs(target);
	const prior = await todosPriorCounts(cwd);
	const { silent, loud, marker } = countByKind(candidates);
	const prevTotal = prior ? prior.silent + prior.loud + prior.marker : 0;
	const total = silent + loud + marker;
	const nw = Math.max(0, total - prevTotal);
	const resolved = Math.max(0, prevTotal - total);
	const report = renderFindings(cwd, candidates, { nw, resolved });

	const candidateList = candidates
		.slice(0, PROMPT_CAP)
		.map((c) => ` - ${relative(cwd, c.path)}:${c.line} [${c.kind}] ${displayCode(c.code)}`);
	if (candidates.length > PROMPT_CAP) {
		candidateList.push(
			` - … and ${candidates.length - PROMPT_CAP} more (truncated for brevity)`,
		);
	}

	return [
		`Inspect the target "${target}" (cwd: ${cwd}) for unfinished work: TODO markers and stub implementations.`,
		`A deterministic pre-scan found ${candidates.length} candidate line(s). Verify each candidate:`,
		...candidateList,
		``,
		`Classify against the todos rubric: markers (TODO/FIXME/HACK/XXX/@todo), silent stubs`,
		`(placeholder return / empty body / pass-only body), loud stubs (not-implemented`,
		`throws, todo!(), TODO("..."), raise NotImplementedError).`,
		`Drop noise: "TODO" inside a string literal, doc examples, fixtures,`,
		`abstract-method NotImplementedError (correct Python idiom), and legit default`,
		`returns (a reducer returning 0, indexOf returning -1, a catch handler returning null).`,
		`Previous run reported: ${
			prior
				? `${prior.silent} silent, ${prior.loud} loud, ${prior.marker} marker`
				: "none (first run)"
		}.`,
		`Write your full verified report to: ${findings}.`,
		`findings.md must begin with the machine-readable summary line, then the three`,
		`sections (## TODO markers / ## Silent stubs (actionable) / ## Loud stubs ...),`,
		`each entry with file:line, evidence, and disposition. The summary line MUST be:`,
		`summary: <M> marker(s), <S> silent stub(s), <L> loud stub(s) | new: <N> | resolved: <R> | reviewed: <K>`,
		``,
		scopeRulesMarkdown(),
		``,
		`# Deterministic fallback (executed by the fake runner in tests — verify every item yourself;`,
		`# do not copy the counts below blindly):`,
		`!write ${findings} ${report}`,
		`!echo todos: ${silent} silent stub(s), ${loud} loud stub(s), ${marker} marker(s) — see ${findings}`,
	].join("\n");
}

/**
 * Build the fix task. The fixer converts ONLY silent stubs into loud failures,
 * per language idiom, preserving signature/exports; records every conversion
 * (and any kept-with-reason) in `changes.md`. Markers are never implemented
 * or deleted; loud stubs are never touched.
 */
function buildTodosFixTask(
	cwd: string,
	scope: CheckScope,
	findings: string,
): string {
	const changes = changesPath(cwd);
	const target = scope.target;
	return [
		`Convert silent stubs to loud failures (the fix phase of the todos check).`,
		`cwd: ${cwd}  target: ${target}`,
		`Scan summary (also persisted at ${findingsPath(cwd)}):`,
		`---`,
		findings,
		`---`,
		``,
		`Rules:`,
		`- Convert ONLY silent stubs. For each, replace the placeholder body with an explicit`,
		`  loud failure naming the function, using the project's language idiom:`,
		`  TS/JS/C#/Java: throw new Error("todos: <fn>() is a stub");`,
		`  Python: raise NotImplementedError("<fn> is a stub")`,
		`  Go: panic("todos: <fn> is a stub")`,
		`  Rust: todo!("<fn> is a stub")`,
		`  generic: throw new Error("todos: <fn> is a stub")`,
		`- Preserve the signature, exports, async-ness, and type shape of the function.`,
		`- Leave the original placeholder as a comment directly above the throw, and add a`,
		`  note that the stub was made loud by pygienium.`,
		`- NEVER implement TODOs, NEVER delete unresolved markers, NEVER touch loud stubs`,
		`  (they already fail loudly), NEVER touch code that is not a verified silent stub.`,
		`- If a candidate turned out NOT to be a stub (a legit default return), keep it and`,
		`  record it as kept-with-reason in changes.md.`,
		`- Apply the smallest possible diff; preserve tests and conventions.`,
		`- Write changes.md to ${changes} listing every conversion (auto) or keep (reason).`,
		``,
		`# Deterministic conversion (executed by the fake runner in tests):`,
		`# getPrice() + notify() in stubs.ts converted from silent placeholders to loud throws.`,
		`# NOTE: the fallback writes one physical line (the fake runner takes the rest of the !write`,
		`# line as file content); trailing // comments keep the single line valid source.`,
		`!write ${target}/stubs.ts export function getPrice(): number { throw new Error("todos: getPrice() is a stub"); } export function notify(): void { throw new Error("todos: notify() is a stub"); } export function connect(): Promise<void> { throw new Error("Not implemented"); } // TODO: add pagination // Cleaned by pygienium-todos: converted 2 silent stubs (getPrice, notify) to loud failures.`,
		`!write ${changes} # TODOs & stubs changes | summary: 2 silent stub(s) converted to loud, 0 kept | ## Converted to loud (auto) | 1. ${target}/stubs.ts:3 — getPrice() — body was \`return 0\` placeholder; now throws \`todos: getPrice() is a stub\` | 2. ${target}/stubs.ts:6 — notify() — body was empty; now throws \`todos: notify() is a stub\` | ## Kept (with reason) | (none)`,
		`!echo todos: 2 silent stub(s) converted — see ${changes}`,
	].join("\n");
}

/**
 * Verify hook: confirms the check actually produced its artifacts — after the
 * scan phase `findings.md` must exist; after the fix phase `changes.md` must
 * exist too. Returns an error string to fail verify, or `undefined` to pass.
 */
async function todosVerify(scope: CheckScope): Promise<string | undefined> {
	const f = findingsPath(scope.cwd);
	try {
		await stat(f);
	} catch {
		return `todos verify: expected findings.md at ${f} after scan, none found.`;
	}
	if (scope.fix) {
		const c = changesPath(scope.cwd);
		try {
			await stat(c);
		} catch {
			return `todos verify: expected changes.md at ${c} after --fix, none found.`;
		}
	}
	return undefined;
}

/** The todos check definition; registers itself on import. */
const todosCheck: CheckDefinition = {
	name: "todos",
	label: "TODOs & stubs",
	description:
		"Inventory TODO/FIXME markers and stub implementations; with --fix, convert silent stubs (placeholder returns, empty bodies) into loud failures — never implementing TODOs.",
	agentName: "todos",
	phaseId: "analysis",
	buildScanTask: buildTodosScanTask,
	buildFixTask: buildTodosFixTask,
	gate: todosGate,
	verify: todosVerify,
};

registerCheck(todosCheck);

export { todosCheck };