0.2.0: per-query analysis, cross-query corroboration, expanded authority scoring
- Per-query result grouping: findings analyzed under the query that produced them, with angle + query provenance carried through to synthesis - Cross-query corroboration via multi-query URL counts (before global dedup); same-query findings no longer corroborate each other - Robust LLM JSON parsing (code fences, trailing commas, prose prefixes) - Expanded domain authority map to 90+ domains + LOW_AUTHORITY_DOMAINS floor; authority-aware source truncation in analysis prompts - Citation integrity: authoritative bibliography replaces LLM references, hallucinated inline citations stripped - Near-duplicate title detection for syndicated articles; cross-round finding dedup with quality-ranked 30-finding synthesis cap - Junk result filtering, retry-with-backoff, fixed successfulSearches count - Standalone harness (scripts/run-harness.ts) for cache-bypassing iteration
This commit is contained in:
115
scripts/run-harness.ts
Normal file
115
scripts/run-harness.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* Deep Research — standalone end-to-end test harness
|
||||
*
|
||||
* Imports the extension's REAL source (fresh, uncached) and runs the full
|
||||
* research pipeline with real Firecrawl + real pi agent sessions.
|
||||
*
|
||||
* WHY: pi caches extension modules per session (keyed by path + cwd +
|
||||
* generation) and only invalidates on /reload or cwd change. Tool calls in
|
||||
* a live session therefore run the version loaded at session start. This
|
||||
* harness bypasses that cache so you can iterate on src/ without reloading
|
||||
* pi, and it prints verification stats (round stats, angle provenance,
|
||||
* corroboration distribution, citation integrity, authority stats).
|
||||
*
|
||||
* Usage (from repo root):
|
||||
* NODE_PATH=/opt/homebrew/lib/node_modules bun scripts/run-harness.ts \
|
||||
* "<question>" [depth] [breadth] [audience]
|
||||
*
|
||||
* Requires the pi SDK to be resolvable (NODE_PATH above points at the
|
||||
* global pi install) and Firecrawl reachable (settings.json firecrawl.baseUrl).
|
||||
* Report is written to /tmp/deepi-harness/report.md.
|
||||
*/
|
||||
import { runDeepResearch } from "../src/research.ts";
|
||||
import type { ResearchReport } from "../src/types.ts";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
|
||||
const question =
|
||||
process.argv[2] ?? "Compare Rust and Go for backend services in 2025";
|
||||
const depth = Number(process.argv[3] ?? 2);
|
||||
const breadth = Number(process.argv[4] ?? 3);
|
||||
const audience = (process.argv[5] ?? "expert") as
|
||||
| "expert"
|
||||
| "general"
|
||||
| "executive";
|
||||
|
||||
const started = Date.now();
|
||||
|
||||
const report: ResearchReport = await runDeepResearch(
|
||||
{
|
||||
question,
|
||||
depth,
|
||||
breadth,
|
||||
format: "markdown",
|
||||
audience,
|
||||
},
|
||||
{ cwd: process.cwd() } as any,
|
||||
(update) => {
|
||||
const round = update.round
|
||||
? ` [r${update.round}/${update.totalRounds}]`
|
||||
: "";
|
||||
console.log(` [${update.phase}${round}] ${update.message}`);
|
||||
},
|
||||
);
|
||||
|
||||
console.log("\n" + "=".repeat(80));
|
||||
console.log(`DURATION: ${((Date.now() - started) / 1000).toFixed(1)}s`);
|
||||
console.log(`TOTAL SEARCHES: ${report.totalSearches}`);
|
||||
console.log(`TOTAL PAGES: ${report.totalPagesScraped}`);
|
||||
console.log(`REFERENCES: ${report.references.length}`);
|
||||
console.log(`ROUNDS: ${report.rounds.length}`);
|
||||
|
||||
for (const round of report.rounds) {
|
||||
console.log(
|
||||
` Round ${round.round}: ${round.queries.length} queries (${round.successfulSearches} ok, ${round.failedSearches} failed) → ${round.results.length} unique pages → ${round.findings.length} findings`,
|
||||
);
|
||||
const angles = new Map<string, number>();
|
||||
for (const f of round.findings) {
|
||||
const a = f.angle ?? "none";
|
||||
angles.set(a, (angles.get(a) ?? 0) + 1);
|
||||
}
|
||||
console.log(
|
||||
` finding angles: ${Array.from(angles.entries())
|
||||
.map(([a, n]) => `${a}(${n})`)
|
||||
.join(", ")}`,
|
||||
);
|
||||
const corr = round.findings.map((f) => f.corroborationScore ?? 0);
|
||||
const strong = corr.filter((c) => c >= 0.5).length;
|
||||
const partial = corr.filter((c) => c > 0 && c < 0.5).length;
|
||||
const none = corr.filter((c) => c === 0).length;
|
||||
console.log(
|
||||
` corroboration: ${strong} strong(>=0.5), ${partial} partial, ${none} none`,
|
||||
);
|
||||
}
|
||||
|
||||
// Citation integrity: citations used in the report BODY vs reference list.
|
||||
// (Reference titles may legitimately contain "[2026]" etc. — those live in
|
||||
// the ## References section which was rebuilt authoritatively.)
|
||||
const refIds = new Set(report.references.map((r) => r.id));
|
||||
const body = report.finalReport.replace(/^## References[\s\S]*$/m, "");
|
||||
const cited = new Set(
|
||||
[...body.matchAll(/\[(\d+)\]/g)].map((m) => Number(m[1])),
|
||||
);
|
||||
const dangling = [...cited].filter((id) => !refIds.has(id));
|
||||
console.log(
|
||||
`CITATIONS (body only): ${cited.size} unique numbers used, ${dangling.length} dangling (${dangling.join(",")})`,
|
||||
);
|
||||
console.log(
|
||||
`REFERENCES SECTION PRESENT: ${/^## References/m.test(report.finalReport)}`,
|
||||
);
|
||||
const iconCount = (report.finalReport.match(/[⭐✓○]/g) ?? []).length;
|
||||
console.log(`AUTHORITY ICONS IN REFERENCES: ${iconCount}`);
|
||||
|
||||
const authorities = report.references.map((r) => r.authorityScore);
|
||||
const avgAuth =
|
||||
authorities.reduce((a, b) => a + b, 0) / Math.max(1, authorities.length);
|
||||
console.log(
|
||||
`AVG SOURCE AUTHORITY: ${(avgAuth * 100).toFixed(0)}% (max ${(Math.max(...authorities) * 100).toFixed(0)}%, min ${(Math.min(...authorities) * 100).toFixed(0)}%)`,
|
||||
);
|
||||
console.log(
|
||||
"DOMAINS:",
|
||||
[...new Set(report.references.map((r) => r.domain))].join(", "),
|
||||
);
|
||||
|
||||
mkdirSync("/tmp/deepi-harness", { recursive: true });
|
||||
writeFileSync("/tmp/deepi-harness/report.md", report.finalReport);
|
||||
console.log("\nReport saved to /tmp/deepi-harness/report.md");
|
||||
Reference in New Issue
Block a user