0.2.0: per-query analysis, cross-query corroboration, expanded authority scoring
- Per-query result grouping: findings analyzed under the query that produced them, with angle + query provenance carried through to synthesis - Cross-query corroboration via multi-query URL counts (before global dedup); same-query findings no longer corroborate each other - Robust LLM JSON parsing (code fences, trailing commas, prose prefixes) - Expanded domain authority map to 90+ domains + LOW_AUTHORITY_DOMAINS floor; authority-aware source truncation in analysis prompts - Citation integrity: authoritative bibliography replaces LLM references, hallucinated inline citations stripped - Near-duplicate title detection for syndicated articles; cross-round finding dedup with quality-ranked 30-finding synthesis cap - Junk result filtering, retry-with-backoff, fixed successfulSearches count - Standalone harness (scripts/run-harness.ts) for cache-bypassing iteration
This commit is contained in:
109
src/report.ts
109
src/report.ts
@@ -15,6 +15,7 @@ import type {
|
||||
Finding,
|
||||
} from "./types";
|
||||
import { runAnalysisAgent } from "./agent";
|
||||
import { isNearDuplicateTitle } from "./firecrawl";
|
||||
|
||||
/** Return shape from synthesizeReport */
|
||||
export interface SynthesisResult {
|
||||
@@ -22,6 +23,26 @@ export interface SynthesisResult {
|
||||
references: Reference[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Maximum findings included in the synthesis prompt.
|
||||
* Keeps token usage bounded and forces the synthesizer to focus on
|
||||
* the highest-quality evidence.
|
||||
*/
|
||||
const MAX_SYNTHESIS_FINDINGS = 30;
|
||||
|
||||
/**
|
||||
* Rank a finding for inclusion in synthesis: authority-weighted,
|
||||
* confidence-weighted, with a corroboration bonus.
|
||||
*/
|
||||
function findingQualityScore(f: Finding): number {
|
||||
const authority =
|
||||
(f.bestSourceAuthority ?? f.avgSourceAuthority ?? 0.5) || 0.5;
|
||||
const confidenceWeight =
|
||||
f.confidence === "high" ? 1.0 : f.confidence === "medium" ? 0.7 : 0.45;
|
||||
const corroborationBonus = (f.corroborationScore ?? 0) * 0.3;
|
||||
return authority * confidenceWeight + corroborationBonus;
|
||||
}
|
||||
|
||||
/* ── System Prompts ──────────────────────────────────────────────── */
|
||||
|
||||
function buildSynthesisSystem(audience: string): string {
|
||||
@@ -93,21 +114,61 @@ function buildEvidenceText(
|
||||
}
|
||||
}
|
||||
|
||||
// Organize findings by thematic angle
|
||||
const evidenceByAngle = new Map<string, Finding[]>();
|
||||
for (const round of rounds) {
|
||||
for (const finding of round.findings) {
|
||||
const angle = round.queries[0]?.angle ?? "technical";
|
||||
if (!evidenceByAngle.has(angle)) evidenceByAngle.set(angle, []);
|
||||
evidenceByAngle.get(angle)!.push(finding);
|
||||
// ── Deduplicate findings across rounds ────────────────────────────
|
||||
// The same claim often surfaces in multiple rounds under slightly
|
||||
// different titles. Merge them (union of sources/quotes, keep the
|
||||
// highest-quality version) so the synthesizer isn't double-counting.
|
||||
const deduped: Finding[] = [];
|
||||
for (const finding of allFindings) {
|
||||
const dupIndex = deduped.findIndex(
|
||||
(f) =>
|
||||
f.title !== finding.title &&
|
||||
isNearDuplicateTitle(f.title, finding.title),
|
||||
);
|
||||
if (dupIndex === -1) {
|
||||
deduped.push({ ...finding });
|
||||
} else {
|
||||
const existing = deduped[dupIndex];
|
||||
deduped[dupIndex] = {
|
||||
title: existing.title,
|
||||
summary: existing.summary,
|
||||
sources: Array.from(new Set([...existing.sources, ...finding.sources])),
|
||||
keyQuotes: Array.from(
|
||||
new Set([...existing.keyQuotes, ...finding.keyQuotes]),
|
||||
).slice(0, 3),
|
||||
confidence:
|
||||
existing.confidence === "high" || finding.confidence === "high"
|
||||
? "high"
|
||||
: existing.confidence === "medium" ||
|
||||
finding.confidence === "medium"
|
||||
? "medium"
|
||||
: "low",
|
||||
query: existing.query ?? finding.query,
|
||||
angle: existing.angle ?? finding.angle,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Rank and cap findings for synthesis ───────────────────────────
|
||||
const ranked = deduped
|
||||
.map((f) => ({ f, score: findingQualityScore(f) }))
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, MAX_SYNTHESIS_FINDINGS)
|
||||
.map(({ f }) => f);
|
||||
|
||||
// Organize findings by their own angle (provenance-aware)
|
||||
const evidenceByAngle = new Map<string, Finding[]>();
|
||||
for (const finding of ranked) {
|
||||
const angle = finding.angle ?? "general";
|
||||
if (!evidenceByAngle.has(angle)) evidenceByAngle.set(angle, []);
|
||||
evidenceByAngle.get(angle)!.push(finding);
|
||||
}
|
||||
|
||||
let evidenceText = `## Research Question\n${question}\n\n`;
|
||||
evidenceText += `## Overview\n- Rounds of research: ${rounds.length}\n`;
|
||||
evidenceText += `- Total searches executed: ${totalSearches}\n`;
|
||||
evidenceText += `- Total pages analyzed: ${totalPages}\n`;
|
||||
evidenceText += `- Key findings extracted: ${allFindings.length}\n\n`;
|
||||
evidenceText += `- Key findings extracted: ${allFindings.length} (${ranked.length} passed dedup/quality filter)\n\n`;
|
||||
|
||||
// Build evidence grouped by angle with reference IDs
|
||||
for (const [angle, findings] of Array.from(evidenceByAngle)) {
|
||||
@@ -199,12 +260,33 @@ Remember to use numbered citations like [1], [2] and include a ## References sec
|
||||
// Build bibliography section
|
||||
const bibSection = buildBibliography(referenceMap);
|
||||
|
||||
// Append references if not already present
|
||||
let report = result.text;
|
||||
if (!report.includes("## References") && !report.includes("# References")) {
|
||||
report += `\n\n${bibSection}`;
|
||||
|
||||
// ── Citation integrity ─────────────────────────────────────────
|
||||
// 1. Strip any references section the LLM wrote and replace it with
|
||||
// the authoritative bibliography (built from real scraped sources).
|
||||
// 2. Remove inline [n] citations that point at IDs outside the
|
||||
// bibliography (hallucinated numbers), so every citation resolves.
|
||||
report = report.replace(
|
||||
/^#+\s*references\s*$/gim,
|
||||
"\n## END_OF_REPORT_MARKER",
|
||||
);
|
||||
const markerIdx = report.indexOf("## END_OF_REPORT_MARKER");
|
||||
if (markerIdx !== -1) {
|
||||
report = report.slice(0, markerIdx).trimEnd();
|
||||
}
|
||||
|
||||
const maxRefId = Math.max(
|
||||
0,
|
||||
...Array.from(referenceMap.values()).map((r) => r.id),
|
||||
);
|
||||
report = report.replace(/\[(\d+)\]/g, (match, id: string) => {
|
||||
const num = parseInt(id, 10);
|
||||
return num >= 1 && num <= maxRefId ? match : "";
|
||||
});
|
||||
|
||||
report = report.trimEnd() + `\n\n${bibSection}`;
|
||||
|
||||
return { report, references: Array.from(referenceMap.values()) };
|
||||
}
|
||||
|
||||
@@ -354,7 +436,7 @@ function generateFallbackReport(
|
||||
const byAngle = new Map<string, Finding[]>();
|
||||
for (const round of rounds) {
|
||||
for (const f of round.findings) {
|
||||
const angle = round.queries[0]?.angle ?? "general";
|
||||
const angle = f.angle ?? round.queries[0]?.angle ?? "general";
|
||||
if (!byAngle.has(angle)) byAngle.set(angle, []);
|
||||
byAngle.get(angle)!.push(f);
|
||||
}
|
||||
@@ -430,7 +512,8 @@ function generateFallbackReport(
|
||||
lines.push("## Methodology");
|
||||
lines.push("");
|
||||
for (const round of rounds) {
|
||||
const failedSearches = round.queries.length - round.successfulSearches;
|
||||
const failedSearches =
|
||||
round.failedSearches ?? round.queries.length - round.successfulSearches;
|
||||
lines.push(`### Round ${round.round}`);
|
||||
lines.push(
|
||||
`Queries: ${round.queries.map((q) => `"${q.query}" [${q.angle}]`).join(", ")}`,
|
||||
|
||||
Reference in New Issue
Block a user