0.2.0: per-query analysis, cross-query corroboration, expanded authority scoring

- Per-query result grouping: findings analyzed under the query that produced
  them, with angle + query provenance carried through to synthesis
- Cross-query corroboration via multi-query URL counts (before global dedup);
  same-query findings no longer corroborate each other
- Robust LLM JSON parsing (code fences, trailing commas, prose prefixes)
- Expanded domain authority map to 90+ domains + LOW_AUTHORITY_DOMAINS floor;
  authority-aware source truncation in analysis prompts
- Citation integrity: authoritative bibliography replaces LLM references,
  hallucinated inline citations stripped
- Near-duplicate title detection for syndicated articles; cross-round finding
  dedup with quality-ranked 30-finding synthesis cap
- Junk result filtering, retry-with-backoff, fixed successfulSearches count
- Standalone harness (scripts/run-harness.ts) for cache-bypassing iteration
This commit is contained in:
2026-08-02 12:49:07 -04:00
parent 4ece83f5c6
commit c3e0770cc4
9 changed files with 707 additions and 135 deletions

115
scripts/run-harness.ts Normal file
View File

@@ -0,0 +1,115 @@
/**
* Deep Research — standalone end-to-end test harness
*
* Imports the extension's REAL source (fresh, uncached) and runs the full
* research pipeline with real Firecrawl + real pi agent sessions.
*
* WHY: pi caches extension modules per session (keyed by path + cwd +
* generation) and only invalidates on /reload or cwd change. Tool calls in
* a live session therefore run the version loaded at session start. This
* harness bypasses that cache so you can iterate on src/ without reloading
* pi, and it prints verification stats (round stats, angle provenance,
* corroboration distribution, citation integrity, authority stats).
*
* Usage (from repo root):
* NODE_PATH=/opt/homebrew/lib/node_modules bun scripts/run-harness.ts \
* "<question>" [depth] [breadth] [audience]
*
* Requires the pi SDK to be resolvable (NODE_PATH above points at the
* global pi install) and Firecrawl reachable (settings.json firecrawl.baseUrl).
* Report is written to /tmp/deepi-harness/report.md.
*/
import { runDeepResearch } from "../src/research.ts";
import type { ResearchReport } from "../src/types.ts";
import { mkdirSync, writeFileSync } from "node:fs";
const question =
process.argv[2] ?? "Compare Rust and Go for backend services in 2025";
const depth = Number(process.argv[3] ?? 2);
const breadth = Number(process.argv[4] ?? 3);
const audience = (process.argv[5] ?? "expert") as
| "expert"
| "general"
| "executive";
const started = Date.now();
const report: ResearchReport = await runDeepResearch(
{
question,
depth,
breadth,
format: "markdown",
audience,
},
{ cwd: process.cwd() } as any,
(update) => {
const round = update.round
? ` [r${update.round}/${update.totalRounds}]`
: "";
console.log(` [${update.phase}${round}] ${update.message}`);
},
);
console.log("\n" + "=".repeat(80));
console.log(`DURATION: ${((Date.now() - started) / 1000).toFixed(1)}s`);
console.log(`TOTAL SEARCHES: ${report.totalSearches}`);
console.log(`TOTAL PAGES: ${report.totalPagesScraped}`);
console.log(`REFERENCES: ${report.references.length}`);
console.log(`ROUNDS: ${report.rounds.length}`);
for (const round of report.rounds) {
console.log(
` Round ${round.round}: ${round.queries.length} queries (${round.successfulSearches} ok, ${round.failedSearches} failed) → ${round.results.length} unique pages → ${round.findings.length} findings`,
);
const angles = new Map<string, number>();
for (const f of round.findings) {
const a = f.angle ?? "none";
angles.set(a, (angles.get(a) ?? 0) + 1);
}
console.log(
` finding angles: ${Array.from(angles.entries())
.map(([a, n]) => `${a}(${n})`)
.join(", ")}`,
);
const corr = round.findings.map((f) => f.corroborationScore ?? 0);
const strong = corr.filter((c) => c >= 0.5).length;
const partial = corr.filter((c) => c > 0 && c < 0.5).length;
const none = corr.filter((c) => c === 0).length;
console.log(
` corroboration: ${strong} strong(>=0.5), ${partial} partial, ${none} none`,
);
}
// Citation integrity: citations used in the report BODY vs reference list.
// (Reference titles may legitimately contain "[2026]" etc. — those live in
// the ## References section which was rebuilt authoritatively.)
const refIds = new Set(report.references.map((r) => r.id));
const body = report.finalReport.replace(/^## References[\s\S]*$/m, "");
const cited = new Set(
[...body.matchAll(/\[(\d+)\]/g)].map((m) => Number(m[1])),
);
const dangling = [...cited].filter((id) => !refIds.has(id));
console.log(
`CITATIONS (body only): ${cited.size} unique numbers used, ${dangling.length} dangling (${dangling.join(",")})`,
);
console.log(
`REFERENCES SECTION PRESENT: ${/^## References/m.test(report.finalReport)}`,
);
const iconCount = (report.finalReport.match(/[⭐✓○]/g) ?? []).length;
console.log(`AUTHORITY ICONS IN REFERENCES: ${iconCount}`);
const authorities = report.references.map((r) => r.authorityScore);
const avgAuth =
authorities.reduce((a, b) => a + b, 0) / Math.max(1, authorities.length);
console.log(
`AVG SOURCE AUTHORITY: ${(avgAuth * 100).toFixed(0)}% (max ${(Math.max(...authorities) * 100).toFixed(0)}%, min ${(Math.min(...authorities) * 100).toFixed(0)}%)`,
);
console.log(
"DOMAINS:",
[...new Set(report.references.map((r) => r.domain))].join(", "),
);
mkdirSync("/tmp/deepi-harness", { recursive: true });
writeFileSync("/tmp/deepi-harness/report.md", report.finalReport);
console.log("\nReport saved to /tmp/deepi-harness/report.md");