From c3e0770cc4d40d524a283753f1cf8416489e9046 Mon Sep 17 00:00:00 2001 From: Michael Freno Date: Sun, 2 Aug 2026 12:49:07 -0400 Subject: [PATCH] 0.2.0: per-query analysis, cross-query corroboration, expanded authority scoring - Per-query result grouping: findings analyzed under the query that produced them, with angle + query provenance carried through to synthesis - Cross-query corroboration via multi-query URL counts (before global dedup); same-query findings no longer corroborate each other - Robust LLM JSON parsing (code fences, trailing commas, prose prefixes) - Expanded domain authority map to 90+ domains + LOW_AUTHORITY_DOMAINS floor; authority-aware source truncation in analysis prompts - Citation integrity: authoritative bibliography replaces LLM references, hallucinated inline citations stripped - Near-duplicate title detection for syndicated articles; cross-round finding dedup with quality-ranked 30-finding synthesis cap - Junk result filtering, retry-with-backoff, fixed successfulSearches count - Standalone harness (scripts/run-harness.ts) for cache-bypassing iteration --- README.md | 14 +- index.ts | 8 +- package.json | 2 +- scripts/run-harness.ts | 115 ++++++++++++++++ src/firecrawl.ts | 189 +++++++++++++++++++++++++- src/queries.ts | 292 +++++++++++++++++++++++++++++------------ src/report.ts | 109 +++++++++++++-- src/research.ts | 105 +++++++++++---- src/types.ts | 8 +- 9 files changed, 707 insertions(+), 135 deletions(-) create mode 100644 scripts/run-harness.ts diff --git a/README.md b/README.md index 0b22d06..7634e98 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,16 @@ pi install npm:@mikefreno/deep-research - **Multi-round iteration**: Each round generates follow-up queries based on previous findings (depth 1-3) - **Parallel query expansion**: Multiple diverse search queries per round (breadth 1-5) covering technical, practical, comparative, critical, and forward-looking angles +- **Sub-question decomposition**: Broad questions are broken into focused sub-topics before query generation (depth > 1) - **Round-robin parallel execution**: Searches and analyses run concurrently within each round using bounded-concurrency worker pools, dramatically reducing total research time -- **LLM-driven analysis**: Each round's results are analyzed by an agent session to extract structured findings with confidence ratings +- **LLM-driven analysis**: Each query's results are analyzed by its own agent session (per-query provenance) to extract structured findings with confidence ratings +- **Source authority scoring**: Every source is scored by domain authority; low-quality SEO domains are penalized with a hard floor; findings are ranked by authority × confidence before synthesis +- **Cross-query corroboration**: A finding is corroborated only when its sources were independently surfaced by multiple different search queries +- **Citation integrity**: References are rebuilt from the authoritative bibliography (never the LLM's), and hallucinated inline citation numbers are stripped +- **Near-duplicate detection**: Syndicated copies of the same article are removed by title similarity, and duplicate findings across rounds are merged - **Automatic deduplication**: Search results are deduplicated by URL across all queries -- **Graceful degradation**: Individual search or analysis failures don't crash the full research — partial results are preserved +- **Robust LLM output parsing**: JSON output with code fences, prose prefixes, or trailing commas is parsed reliably +- **Graceful degradation**: Individual search or analysis failures don't crash the full research — partial results are preserved, with retry-with-backoff for transient Firecrawl errors - **Progress streaming**: Real-time progress widget with spinner, phase indicators, and progress bar - **Abort support**: Research can be cancelled mid-flight via `AbortSignal` - **Rich TUI rendering**: Compact collapsed view and detailed expanded view in the terminal UI @@ -30,13 +36,15 @@ deep_research — multi-round deep web research via Firecrawl with iterative que ``` Parameters: + | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `question` | string | — | The research question to investigate | | `depth` | integer (1-3) | 2 | Number of research rounds | | `breadth` | integer (1-5) | 3 | Search queries per round | | `format` | "markdown" \| "structured" | "markdown" | Output format for the report | -| `details.showRoundDetails` | boolean | false | Include per-round search metadata in output | +| `audience` | "general" \| "expert" \| "executive" | "general" | Tone and depth for the report audience | +| `details.showRoundDetails` | boolean | false | Include per-round search metadata (incl. failed searches) in output | ### Command (interactive) diff --git a/index.ts b/index.ts index b4bd1cb..4663a27 100644 --- a/index.ts +++ b/index.ts @@ -122,6 +122,7 @@ interface ResearchDetails { queries: string[]; findingsCount: number; resultsCount: number; + failedSearches: number; }>; totalSearches: number; totalPagesScraped: number; @@ -288,6 +289,7 @@ export default function (pi: ExtensionAPI) { queries: r.queries.map((q) => q.query), findingsCount: r.findings.length, resultsCount: r.results.length, + failedSearches: r.failedSearches, })), totalSearches: researchResult.totalSearches, totalPagesScraped: researchResult.totalPagesScraped, @@ -326,7 +328,11 @@ export default function (pi: ExtensionAPI) { output += `- "${q.query}" (${q.angle}) — ${q.rationale}\n`; } output += `\n**Results scraped:** ${round.results.length}\n`; - output += `**Findings extracted:** ${round.findings.length}\n\n`; + output += `**Findings extracted:** ${round.findings.length}\n`; + if (round.failedSearches > 0) { + output += `**Failed searches:** ${round.failedSearches}\n`; + } + output += `\n`; } output += `**Total searches:** ${researchResult.totalSearches}\n`; output += `**Total pages scraped:** ${researchResult.totalPagesScraped}\n`; diff --git a/package.json b/package.json index e989816..bd35294 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@mikefreno/deepi-research", - "version": "0.1.2", + "version": "0.2.0", "description": "Deep research extension for pi — parallel web research via Firecrawl with iterative query refinement", "keywords": [ "pi-package", diff --git a/scripts/run-harness.ts b/scripts/run-harness.ts new file mode 100644 index 0000000..4c39b2f --- /dev/null +++ b/scripts/run-harness.ts @@ -0,0 +1,115 @@ +/** + * Deep Research — standalone end-to-end test harness + * + * Imports the extension's REAL source (fresh, uncached) and runs the full + * research pipeline with real Firecrawl + real pi agent sessions. + * + * WHY: pi caches extension modules per session (keyed by path + cwd + + * generation) and only invalidates on /reload or cwd change. Tool calls in + * a live session therefore run the version loaded at session start. This + * harness bypasses that cache so you can iterate on src/ without reloading + * pi, and it prints verification stats (round stats, angle provenance, + * corroboration distribution, citation integrity, authority stats). + * + * Usage (from repo root): + * NODE_PATH=/opt/homebrew/lib/node_modules bun scripts/run-harness.ts \ + * "" [depth] [breadth] [audience] + * + * Requires the pi SDK to be resolvable (NODE_PATH above points at the + * global pi install) and Firecrawl reachable (settings.json firecrawl.baseUrl). + * Report is written to /tmp/deepi-harness/report.md. + */ +import { runDeepResearch } from "../src/research.ts"; +import type { ResearchReport } from "../src/types.ts"; +import { mkdirSync, writeFileSync } from "node:fs"; + +const question = + process.argv[2] ?? "Compare Rust and Go for backend services in 2025"; +const depth = Number(process.argv[3] ?? 2); +const breadth = Number(process.argv[4] ?? 3); +const audience = (process.argv[5] ?? "expert") as + | "expert" + | "general" + | "executive"; + +const started = Date.now(); + +const report: ResearchReport = await runDeepResearch( + { + question, + depth, + breadth, + format: "markdown", + audience, + }, + { cwd: process.cwd() } as any, + (update) => { + const round = update.round + ? ` [r${update.round}/${update.totalRounds}]` + : ""; + console.log(` [${update.phase}${round}] ${update.message}`); + }, +); + +console.log("\n" + "=".repeat(80)); +console.log(`DURATION: ${((Date.now() - started) / 1000).toFixed(1)}s`); +console.log(`TOTAL SEARCHES: ${report.totalSearches}`); +console.log(`TOTAL PAGES: ${report.totalPagesScraped}`); +console.log(`REFERENCES: ${report.references.length}`); +console.log(`ROUNDS: ${report.rounds.length}`); + +for (const round of report.rounds) { + console.log( + ` Round ${round.round}: ${round.queries.length} queries (${round.successfulSearches} ok, ${round.failedSearches} failed) → ${round.results.length} unique pages → ${round.findings.length} findings`, + ); + const angles = new Map(); + for (const f of round.findings) { + const a = f.angle ?? "none"; + angles.set(a, (angles.get(a) ?? 0) + 1); + } + console.log( + ` finding angles: ${Array.from(angles.entries()) + .map(([a, n]) => `${a}(${n})`) + .join(", ")}`, + ); + const corr = round.findings.map((f) => f.corroborationScore ?? 0); + const strong = corr.filter((c) => c >= 0.5).length; + const partial = corr.filter((c) => c > 0 && c < 0.5).length; + const none = corr.filter((c) => c === 0).length; + console.log( + ` corroboration: ${strong} strong(>=0.5), ${partial} partial, ${none} none`, + ); +} + +// Citation integrity: citations used in the report BODY vs reference list. +// (Reference titles may legitimately contain "[2026]" etc. — those live in +// the ## References section which was rebuilt authoritatively.) +const refIds = new Set(report.references.map((r) => r.id)); +const body = report.finalReport.replace(/^## References[\s\S]*$/m, ""); +const cited = new Set( + [...body.matchAll(/\[(\d+)\]/g)].map((m) => Number(m[1])), +); +const dangling = [...cited].filter((id) => !refIds.has(id)); +console.log( + `CITATIONS (body only): ${cited.size} unique numbers used, ${dangling.length} dangling (${dangling.join(",")})`, +); +console.log( + `REFERENCES SECTION PRESENT: ${/^## References/m.test(report.finalReport)}`, +); +const iconCount = (report.finalReport.match(/[⭐✓○]/g) ?? []).length; +console.log(`AUTHORITY ICONS IN REFERENCES: ${iconCount}`); + +const authorities = report.references.map((r) => r.authorityScore); +const avgAuth = + authorities.reduce((a, b) => a + b, 0) / Math.max(1, authorities.length); +console.log( + `AVG SOURCE AUTHORITY: ${(avgAuth * 100).toFixed(0)}% (max ${(Math.max(...authorities) * 100).toFixed(0)}%, min ${(Math.min(...authorities) * 100).toFixed(0)}%)`, +); +console.log( + "DOMAINS:", + [...new Set(report.references.map((r) => r.domain))].join(", "), +); + +mkdirSync("/tmp/deepi-harness", { recursive: true }); +writeFileSync("/tmp/deepi-harness/report.md", report.finalReport); +console.log("\nReport saved to /tmp/deepi-harness/report.md"); diff --git a/src/firecrawl.ts b/src/firecrawl.ts index c087513..173c864 100644 --- a/src/firecrawl.ts +++ b/src/firecrawl.ts @@ -85,6 +85,51 @@ const AUTHORITY_DOMAINS: Record = { "kubernetes.io": 0.85, "react.dev": 0.85, "nextjs.org": 0.8, + // Official language/platform docs + "go.dev": 0.9, + "golang.org": 0.9, + "rust-lang.org": 0.9, + "nodejs.org": 0.85, + "python.org": 0.85, + "typescriptlang.org": 0.85, + "openai.com": 0.8, + "anthropic.com": 0.8, + "cloud.google.com": 0.8, + "aws.amazon.com": 0.8, + "azure.microsoft.com": 0.8, + "postgresql.org": 0.85, + "sqlite.org": 0.85, + "redis.io": 0.85, + "docker.com": 0.75, + "elastic.co": 0.75, + "grafana.com": 0.75, + "datadoghq.com": 0.75, + "cloudflare.com": 0.8, + "blog.cloudflare.com": 0.8, + "techempower.com": 0.8, + "goframe.org": 0.75, + "corrode.dev": 0.6, + "evrone.com": 0.4, + "rustify.rs": 0.4, + "core.cz": 0.4, + // Medical / clinical + "mayoclinic.org": 0.9, + "heart.org": 0.85, + "researchgate.net": 0.6, + "healthline.com": 0.5, + "medicalnewstoday.com": 0.5, + "webmd.com": 0.45, + "verywellhealth.com": 0.5, + // Databases & dev tools + "mysql.com": 0.85, + "mariadb.org": 0.85, + "cockroachlabs.com": 0.7, + "timescale.com": 0.7, + "mongodb.com": 0.8, + "liquibase.com": 0.6, + "sqlpipe.com": 0.5, + "data-tune.com": 0.4, + "binaryigor.com": 0.4, // Government & non-profits ".gov": 0.9, ".edu": 0.85, @@ -106,6 +151,51 @@ const AUTHORITY_DOMAINS: Record = { "arstechnica.com": 0.65, "wired.com": 0.65, "infoworld.com": 0.55, + // Practitioner/aggregator content with measurable quality + "github.io": 0.6, + "crates.io": 0.7, + "docs.rs": 0.75, + "digitalocean.com": 0.6, + "freecodecamp.org": 0.6, + "geeksforgeeks.org": 0.35, + "stackexchange.com": 0.65, + "huggingface.co": 0.65, + "nasa.gov": 0.9, + "mit.edu": 0.9, + "stanford.edu": 0.9, + "harvard.edu": 0.9, + "ox.ac.uk": 0.9, + "cam.ac.uk": 0.9, + // Low-authority: personal social / SEO content + "linkedin.com": 0.25, + "reddit.com": 0.25, + "x.com": 0.3, + "twitter.com": 0.3, + "youtube.com": 0.3, + "blogspot.com": 0.25, + "substack.com": 0.3, + "hashnode.dev": 0.35, + "quora.com": 0.3, + "netguru.com": 0.3, + "relisoftware.com": 0.3, + "dasroot.net": 0.3, + "devgenius.io": 0.3, + "devnewsletter.com": 0.3, +}; + +/** + * Known low-quality SEO/comparison-spam domains. Content is often + * auto-generated, republished from other sites, or thin on substance. + * These get a hard authority floor so they never rank above real content. + */ +const LOW_AUTHORITY_DOMAINS: Record = { + "markaicode.com": 0.15, + "bytegoblin.io": 0.2, + "towardsdev.com": 0.2, + "rustvsgo.com": 0.3, + "seekingalpha.com": 0.3, + "investopedia.com": 0.55, + "devops-daily.com": 0.3, }; /** Content-type hints based on domain patterns */ @@ -150,6 +240,10 @@ function extractDomain(url: string): string { } function computeAuthorityScore(domain: string): number { + // Hard floor for known low-authority domains first + if (LOW_AUTHORITY_DOMAINS[domain] !== undefined) + return LOW_AUTHORITY_DOMAINS[domain]; + // Direct match first if (AUTHORITY_DOMAINS[domain]) return AUTHORITY_DOMAINS[domain]; @@ -164,6 +258,9 @@ function computeAuthorityScore(domain: string): number { return AUTHORITY_DOMAINS[parent] * 0.9; } + // github.io personal sites: treat as practitioner content (medium) + if (domain.endsWith(".github.io")) return 0.55; + return 0.3; // Unknown / low-authority default } @@ -193,6 +290,47 @@ function tryParseDate(dateStr: string | undefined | null): Date | null { return isNaN(d.getTime()) ? null : d; } +/** + * Normalize a title for near-duplicate detection: lowercase, strip + * punctuation, collapse whitespace, drop common filler words. + * Two syndicated copies of the same article normalize identically. + */ +export function normalizeTitle(title: string): string { + return title + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .replace( + /\b(?:the|a|an|of|for|and|or|in|on|with|vs|versus|to|how|what|why|2024|2025|2026)\b/g, + " ", + ) + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Near-duplicate check between two titles: normalized forms must share + * a substantial token overlap (same core words in the same order). + */ +export function isNearDuplicateTitle(a: string, b: string): boolean { + const normA = normalizeTitle(a); + const normB = normalizeTitle(b); + if (!normA || !normB) return false; + if (normA === normB) return true; + + const tokensA = normA.split(" "); + const tokensB = normB.split(" "); + if (tokensA.length < 3 || tokensB.length < 3) return normA === normB; + + // Check if one title is a substring of the other (after normalization) + if (normA.includes(normB) || normB.includes(normA)) return true; + + // Jaccard-ish overlap on the shorter token set + const [short, long] = + tokensA.length <= tokensB.length ? [tokensA, tokensB] : [tokensB, tokensA]; + const overlap = short.filter((t) => long.includes(t)).length; + return overlap / short.length >= 0.75; +} + /** * Enrich a raw search result with source authority metadata. * Accepts extra fields (e.g. date) from the Firecrawl API response. @@ -241,6 +379,44 @@ async function firecrawlRequest( return res.json(); } +/** + * firecrawlRequest with retry-with-backoff for transient failures + * (429 rate limits, 5xx server errors, network blips). Does NOT retry + * 4xx client errors (invalid requests) or aborts. + */ +async function firecrawlRequestWithRetry( + endpoint: string, + body: Record, + signal?: AbortSignal, + retries: number = 2, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt <= retries; attempt++) { + if (signal?.aborted) throw new Error("Aborted"); + try { + return await firecrawlRequest(endpoint, body, signal); + } catch (error) { + lastError = error; + const status = + error instanceof Error + ? Number(/failed \((\d+)\)/.exec(error.message)?.[1] ?? 0) + : 0; + // Don't retry aborts or 4xx client errors (other than 429) + if ( + signal?.aborted || + (status >= 400 && status < 500 && status !== 429) + ) { + throw error; + } + if (attempt < retries) { + const delayMs = 400 * 2 ** attempt + Math.random() * 200; + await new Promise((r) => setTimeout(r, delayMs)); + } + } + } + throw lastError; +} + export async function isFirecrawlReachable(): Promise { try { const res = await fetch(`${BASE_URL}/v1/scrape`, { @@ -278,7 +454,7 @@ export async function searchWeb( }, }; - const result = await firecrawlRequest("search", body, signal); + const result = await firecrawlRequestWithRetry("search", body, signal); if (!result || typeof result !== "object") return []; @@ -299,7 +475,14 @@ export async function searchWeb( // Preserve extra fields for date extraction ...doc, })) - .filter((r) => r.markdown || r.description); + .filter((r) => { + // Keep results with a meaningful body OR a substantive description. + // Filters out stub pages / pure navigation results that would + // waste analysis tokens. + const hasBody = (r.markdown ?? "").trim().length >= 150; + const hasSubstantiveDesc = (r.description ?? "").trim().length >= 40; + return hasBody || hasSubstantiveDesc; + }); // Enrich each result with source metadata return rawResults.map(enrichResult); @@ -314,7 +497,7 @@ export async function scrapeUrl( url: string, signal?: AbortSignal, ): Promise<{ title: string; markdown: string; links: string[] } | null> { - const result = await firecrawlRequest( + const result = await firecrawlRequestWithRetry( "scrape", { url, formats: ["markdown"] }, signal, diff --git a/src/queries.ts b/src/queries.ts index 4fb6b79..72b639e 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -58,16 +58,73 @@ Look for: - Claims made without sufficient evidence - Conflicting information that needs resolution - Angles that haven't been explored yet -- Missing authoritative sources +- Missing authoritative sources (papers, official docs, primary data) - Practical implications that need more detail - Recent developments that might have updated findings +Guidelines: +- Do NOT repeat or paraphrase queries already explored — aim for genuinely new angles +- Prefer querying for authoritative/primary sources over more blog posts when evidence is weak +- When findings conflict, craft a query designed to resolve the contradiction +- Keep queries concise and keyword-rich + Output ONLY a JSON array of objects with fields: - "query": the search query string - "rationale": what gap this query fills or what angle it explores - "angle": one of "technical" | "practical" | "comparative" | "critical" | "forward-looking" | "authoritative" | "historical" | "case-study" | "data-statistics" | "ethical" `; +/* ── JSON parsing helpers ────────────────────────────────────────── */ + +/** + * Robustly parse a JSON array from LLM output. + * + * LLMs frequently wrap JSON in ```json fences, prepend prose like + * "Here are the queries:", or emit trailing punctuation. This strips + * fences and extracts the first bracketed array before parsing. + * + * Returns null when no array can be extracted. + */ +function parseJsonArray(text: string): unknown[] | null { + if (!text) return null; + + // Strip markdown code fences + const withoutFences = text + .replace(/```(?:json|javascript)?\s*/gi, "") + .replace(/```/g, ""); + + // Find the first '[' ... ']' block (arrays are our target shape) + const start = withoutFences.indexOf("["); + const end = withoutFences.lastIndexOf("]"); + if (start === -1 || end === -1 || end <= start) return null; + + const candidate = withoutFences.slice(start, end + 1); + try { + const parsed = JSON.parse(candidate); + return Array.isArray(parsed) ? parsed : null; + } catch { + // Try to salvage: strip trailing commas (common LLM artifact) + try { + const fixed = candidate.replace(/,\s*([}\]])/g, "$1"); + const parsed = JSON.parse(fixed); + return Array.isArray(parsed) ? parsed : null; + } catch { + return null; + } + } +} + +/** Map a parsed array entry to a SearchQuery, tolerating missing fields. */ +function toSearchQuery(q: Record): SearchQuery | null { + const query = String(q.query ?? "").trim(); + if (!query) return null; + return { + query, + rationale: String(q.rationale ?? "").trim(), + angle: String(q.angle ?? "technical").trim() || "technical", + }; +} + /* ── Sub-Question Decomposition ───────────────────────────────────── */ /** @@ -93,13 +150,13 @@ export async function decomposeQuestion( if (!result.success || !result.text) return []; - try { - const parsed = JSON.parse(result.text); - if (Array.isArray(parsed) && parsed.length > 0) { - return parsed.map(String).filter((s: string) => s.length > 10); - } - } catch { - // parse failed + const parsed = parseJsonArray(result.text); + if (parsed) { + const subQuestions = parsed + .map(String) + .map((s: string) => s.trim()) + .filter((s: string) => s.length > 10); + if (subQuestions.length > 0) return subQuestions; } return []; @@ -140,21 +197,13 @@ export async function generateQueries( if (!result.success || !result.text) continue; - try { - const parsed = JSON.parse(result.text); - if (Array.isArray(parsed)) { - const queries = parsed - .slice(0, queriesPerSub) - .map((q: Record) => ({ - query: String(q.query ?? ""), - rationale: String(q.rationale ?? ""), - angle: String(q.angle ?? "technical"), - })) - .filter((q: { query: string }) => q.query.length > 0); - allQueries.push(...queries); - } - } catch { - // parse failed for this sub-question, continue + const parsed = parseJsonArray(result.text); + if (parsed) { + const queries = parsed + .slice(0, queriesPerSub) + .map((q) => toSearchQuery(q as Record)) + .filter((q): q is SearchQuery => q !== null); + allQueries.push(...queries); } } @@ -182,16 +231,12 @@ Generate ${count} diverse search queries to research this topic effectively. Cov } try { - const parsed = JSON.parse(result.text); - if (Array.isArray(parsed) && parsed.length > 0) { + const parsed = parseJsonArray(result.text); + if (parsed && parsed.length > 0) { return parsed .slice(0, count) - .map((q: Record) => ({ - query: String(q.query ?? ""), - rationale: String(q.rationale ?? ""), - angle: String(q.angle ?? "technical"), - })) - .filter((q: { query: string }) => q.query.length > 0); + .map((q) => toSearchQuery(q as Record)) + .filter((q): q is SearchQuery => q !== null); } } catch { // JSON parse failed, fall back @@ -245,7 +290,7 @@ ${findingsSummary} ${gaps ? `Remaining knowledge gaps:\n${gaps}` : ""} -Generate ${count} follow-up search queries to fill remaining gaps and deepen the research.`; +Generate ${count} follow-up search queries to fill remaining gaps and deepen the research. Do not repeat or paraphrase the queries already explored.`; const result = await runAnalysisAgent( FOLLOWUP_SYSTEM, @@ -260,25 +305,41 @@ Generate ${count} follow-up search queries to fill remaining gaps and deepen the return []; } - try { - const parsed = JSON.parse(result.text); - if (Array.isArray(parsed) && parsed.length > 0) { - return parsed - .slice(0, count) - .map((q: Record) => ({ - query: String(q.query ?? ""), - rationale: String(q.rationale ?? ""), - angle: String(q.angle ?? "technical"), - })) - .filter((q: { query: string }) => q.query.length > 0); + const exploredNormalized = new Set( + rounds.flatMap((r) => r.queries).map((q) => normalizeQueryText(q.query)), + ); + + const parsed = parseJsonArray(result.text); + if (parsed && parsed.length > 0) { + const fresh: SearchQuery[] = []; + for (const q of parsed.slice(0, count)) { + const sq = toSearchQuery(q as Record); + if (!sq) continue; + const normalized = normalizeQueryText(sq.query); + // Skip queries that are near-duplicates of already-explored ones + if (exploredNormalized.has(normalized)) continue; + if (fresh.some((fq) => normalizeQueryText(fq.query) === normalized)) + continue; + exploredNormalized.add(normalized); + fresh.push(sq); } - } catch { - // parse failed + return fresh; } return []; } +/** + * Lightweight query-text normalization for duplicate detection. + */ +function normalizeQueryText(query: string): string { + return query + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + /* ── Fallback Query Generation ────────────────────────────────────── */ /** @@ -316,12 +377,19 @@ function generateFallbackQueries( const ANALYZE_SYSTEM = `You are a research analyst. Given search results for a specific query, extract key findings. For each finding: -- Give it a concise title -- Summarize what was found in 1-3 sentences +- Give it a concise, specific title (a claim, not a topic) +- Summarize what was found in 1-3 sentences, focused on evidence - List which source URLs support this finding - Include 1-2 key quotes from the sources - Rate your confidence (high/medium/low) based on source authority and consistency +Guidelines: +- Extract 3-6 findings maximum, prioritizing the most decision-relevant +- Prefer findings with concrete evidence over generic observations +- Ignore boilerplate, navigation text, and irrelevant tangents in the content +- Do NOT invent quotes — only use text that appears in the provided content +- When sources conflict, note the conflict in the summary + Output ONLY a JSON array of objects with fields: - "title": concise finding title - "summary": 1-3 sentence summary @@ -337,16 +405,28 @@ export async function analyzeResults( results: EnrichedSearchResult[], cwd: string, signal?: AbortSignal, + angle?: string, ): Promise { - // Include authority metadata in the prompt so the LLM can consider source quality + // Include authority metadata in the prompt so the LLM can consider source quality. + // Token budget: give high-authority sources generous space, truncate + // low-authority/SEO content aggressively so junk doesn't dominate the prompt. + const MAX_CHARS_HIGH_AUTH = 3500; + const MAX_CHARS_LOW_AUTH = 1200; + const resultsText = results - .map( - (r, i) => - `--- Result ${i + 1} ---\nTitle: ${r.title}\nURL: ${r.url}\nDomain: ${r.domain}\nAuthority Score: ${(r.authorityScore * 100).toFixed(0)}%\nContent Type: ${r.contentType}\nDescription: ${r.description}\nContent:\n${r.markdown.slice(0, 3000)}`, - ) + .map((r, i) => { + const maxChars = + r.authorityScore >= 0.6 ? MAX_CHARS_HIGH_AUTH : MAX_CHARS_LOW_AUTH; + const content = r.markdown.slice(0, maxChars).trim(); + const body = + content.length > 0 + ? content + : `(no body content; description only)\n${r.description}`; + return `--- Result ${i + 1} ---\nTitle: ${r.title}\nURL: ${r.url}\nDomain: ${r.domain}\nAuthority Score: ${(r.authorityScore * 100).toFixed(0)}%\nContent Type: ${r.contentType}\nDescription: ${r.description}\nContent:\n${body}`; + }) .join("\n\n"); - const taskPrompt = `Search query: "${query}" + const taskPrompt = `Search query: "${query}"${angle ? ` (angle: ${angle})` : ""} Search results: ${resultsText} @@ -364,25 +444,31 @@ Extract key findings from these results. Consider source authority when rating c if (!result.success || !result.text) return []; - try { - const parsed = JSON.parse(result.text); - if (Array.isArray(parsed)) { - return parsed - .map((f: Record) => ({ - title: String(f.title ?? ""), - summary: String(f.summary ?? ""), - sources: Array.isArray(f.sources) ? f.sources.map(String) : [], - keyQuotes: Array.isArray(f.keyQuotes) ? f.keyQuotes.map(String) : [], - confidence: (["high", "medium", "low"].includes(String(f.confidence)) - ? String(f.confidence) + const parsed = parseJsonArray(result.text); + if (parsed) { + return parsed + .map((f) => { + const entry = f as Record; + return { + title: String(entry.title ?? "").trim(), + summary: String(entry.summary ?? "").trim(), + sources: Array.isArray(entry.sources) + ? entry.sources.map(String) + : [], + keyQuotes: Array.isArray(entry.keyQuotes) + ? entry.keyQuotes.map(String) + : [], + confidence: (["high", "medium", "low"].includes( + String(entry.confidence), + ) + ? String(entry.confidence) : "medium") as Finding["confidence"], - })) - .filter( - (f: { title: string; summary: string }) => f.title && f.summary, - ); - } - } catch { - // parse failed + // Provenance: which query and angle produced this finding + query, + angle, + }; + }) + .filter((f) => f.title && f.summary); } return []; @@ -401,7 +487,10 @@ Extract key findings from these results. Consider source authority when rating c * Returns the findings with added corroborationScore, bestSourceAuthority, * and avgSourceAuthority. */ -export function computeCorroboration(findings: Finding[]): Finding[] { +export function computeCorroboration( + findings: Finding[], + urlQueryCounts?: Map, +): Finding[] { if (findings.length === 0) return []; // Collect all unique source URLs and their authority scores @@ -447,23 +536,54 @@ export function computeCorroboration(findings: Finding[]): Finding[] { const avgAuthority = authorities.reduce((a, b) => a + b, 0) / authorities.length; - // Compute corroboration: how many other findings share source URLs - let corroboratingFindings = 0; - const mySources = new Set(finding.sources); + // Compute corroboration. + // + // PRIMARY signal (when urlQueryCounts is provided): what fraction of + // this finding's sources were independently surfaced by multiple + // DIFFERENT search queries? A source found by several independent + // searches is genuinely corroborated; same-query duplicates do not + // count (findings from one query analyzed the same result set). + // + // FALLBACK signal (no map): domain-level agreement across findings + // from different queries. + let corroborationScore: number; - for (const other of findings) { - if (other === finding) continue; - const overlap = other.sources.some((url) => mySources.has(url)); - if (overlap) corroboratingFindings++; + if (urlQueryCounts && urlQueryCounts.size > 0) { + const multiQuerySources = finding.sources.filter( + (url) => (urlQueryCounts.get(url) ?? 1) > 1, + ).length; + corroborationScore = + finding.sources.length > 0 + ? multiQuerySources / finding.sources.length + : 0; + } else { + // Fallback: cross-query agreement by shared domain + const myDomains = new Set( + finding.sources.map((u) => extractDomainSimple(u)), + ); + let corroboratingFindings = 0; + let independentOthers = 0; + + for (const other of findings) { + if (other === finding) continue; + // Same query provenance = same analyzed result set = not independent + if (other.query && finding.query && other.query === finding.query) { + continue; + } + independentOthers++; + const otherDomains = new Set( + other.sources.map((u) => extractDomainSimple(u)), + ); + const shared = [...myDomains].some((d) => otherDomains.has(d)); + if (shared) corroboratingFindings++; + } + + corroborationScore = + independentOthers > 0 + ? Math.min(1, corroboratingFindings / independentOthers) + : 0; } - // Normalize corroboration: 0-1 based on what fraction of other findings agree - const maxCorroboration = findings.length - 1; - const corroborationScore = - maxCorroboration > 0 - ? Math.min(1, corroboratingFindings / maxCorroboration) - : 0; - return { ...finding, corroborationScore: Math.round(corroborationScore * 100) / 100, diff --git a/src/report.ts b/src/report.ts index 882a426..7a15c7c 100644 --- a/src/report.ts +++ b/src/report.ts @@ -15,6 +15,7 @@ import type { Finding, } from "./types"; import { runAnalysisAgent } from "./agent"; +import { isNearDuplicateTitle } from "./firecrawl"; /** Return shape from synthesizeReport */ export interface SynthesisResult { @@ -22,6 +23,26 @@ export interface SynthesisResult { references: Reference[]; } +/** + * Maximum findings included in the synthesis prompt. + * Keeps token usage bounded and forces the synthesizer to focus on + * the highest-quality evidence. + */ +const MAX_SYNTHESIS_FINDINGS = 30; + +/** + * Rank a finding for inclusion in synthesis: authority-weighted, + * confidence-weighted, with a corroboration bonus. + */ +function findingQualityScore(f: Finding): number { + const authority = + (f.bestSourceAuthority ?? f.avgSourceAuthority ?? 0.5) || 0.5; + const confidenceWeight = + f.confidence === "high" ? 1.0 : f.confidence === "medium" ? 0.7 : 0.45; + const corroborationBonus = (f.corroborationScore ?? 0) * 0.3; + return authority * confidenceWeight + corroborationBonus; +} + /* ── System Prompts ──────────────────────────────────────────────── */ function buildSynthesisSystem(audience: string): string { @@ -93,21 +114,61 @@ function buildEvidenceText( } } - // Organize findings by thematic angle - const evidenceByAngle = new Map(); - for (const round of rounds) { - for (const finding of round.findings) { - const angle = round.queries[0]?.angle ?? "technical"; - if (!evidenceByAngle.has(angle)) evidenceByAngle.set(angle, []); - evidenceByAngle.get(angle)!.push(finding); + // ── Deduplicate findings across rounds ──────────────────────────── + // The same claim often surfaces in multiple rounds under slightly + // different titles. Merge them (union of sources/quotes, keep the + // highest-quality version) so the synthesizer isn't double-counting. + const deduped: Finding[] = []; + for (const finding of allFindings) { + const dupIndex = deduped.findIndex( + (f) => + f.title !== finding.title && + isNearDuplicateTitle(f.title, finding.title), + ); + if (dupIndex === -1) { + deduped.push({ ...finding }); + } else { + const existing = deduped[dupIndex]; + deduped[dupIndex] = { + title: existing.title, + summary: existing.summary, + sources: Array.from(new Set([...existing.sources, ...finding.sources])), + keyQuotes: Array.from( + new Set([...existing.keyQuotes, ...finding.keyQuotes]), + ).slice(0, 3), + confidence: + existing.confidence === "high" || finding.confidence === "high" + ? "high" + : existing.confidence === "medium" || + finding.confidence === "medium" + ? "medium" + : "low", + query: existing.query ?? finding.query, + angle: existing.angle ?? finding.angle, + }; } } + // ── Rank and cap findings for synthesis ─────────────────────────── + const ranked = deduped + .map((f) => ({ f, score: findingQualityScore(f) })) + .sort((a, b) => b.score - a.score) + .slice(0, MAX_SYNTHESIS_FINDINGS) + .map(({ f }) => f); + + // Organize findings by their own angle (provenance-aware) + const evidenceByAngle = new Map(); + for (const finding of ranked) { + const angle = finding.angle ?? "general"; + if (!evidenceByAngle.has(angle)) evidenceByAngle.set(angle, []); + evidenceByAngle.get(angle)!.push(finding); + } + let evidenceText = `## Research Question\n${question}\n\n`; evidenceText += `## Overview\n- Rounds of research: ${rounds.length}\n`; evidenceText += `- Total searches executed: ${totalSearches}\n`; evidenceText += `- Total pages analyzed: ${totalPages}\n`; - evidenceText += `- Key findings extracted: ${allFindings.length}\n\n`; + evidenceText += `- Key findings extracted: ${allFindings.length} (${ranked.length} passed dedup/quality filter)\n\n`; // Build evidence grouped by angle with reference IDs for (const [angle, findings] of Array.from(evidenceByAngle)) { @@ -199,12 +260,33 @@ Remember to use numbered citations like [1], [2] and include a ## References sec // Build bibliography section const bibSection = buildBibliography(referenceMap); - // Append references if not already present let report = result.text; - if (!report.includes("## References") && !report.includes("# References")) { - report += `\n\n${bibSection}`; + + // ── Citation integrity ───────────────────────────────────────── + // 1. Strip any references section the LLM wrote and replace it with + // the authoritative bibliography (built from real scraped sources). + // 2. Remove inline [n] citations that point at IDs outside the + // bibliography (hallucinated numbers), so every citation resolves. + report = report.replace( + /^#+\s*references\s*$/gim, + "\n## END_OF_REPORT_MARKER", + ); + const markerIdx = report.indexOf("## END_OF_REPORT_MARKER"); + if (markerIdx !== -1) { + report = report.slice(0, markerIdx).trimEnd(); } + const maxRefId = Math.max( + 0, + ...Array.from(referenceMap.values()).map((r) => r.id), + ); + report = report.replace(/\[(\d+)\]/g, (match, id: string) => { + const num = parseInt(id, 10); + return num >= 1 && num <= maxRefId ? match : ""; + }); + + report = report.trimEnd() + `\n\n${bibSection}`; + return { report, references: Array.from(referenceMap.values()) }; } @@ -354,7 +436,7 @@ function generateFallbackReport( const byAngle = new Map(); for (const round of rounds) { for (const f of round.findings) { - const angle = round.queries[0]?.angle ?? "general"; + const angle = f.angle ?? round.queries[0]?.angle ?? "general"; if (!byAngle.has(angle)) byAngle.set(angle, []); byAngle.get(angle)!.push(f); } @@ -430,7 +512,8 @@ function generateFallbackReport( lines.push("## Methodology"); lines.push(""); for (const round of rounds) { - const failedSearches = round.queries.length - round.successfulSearches; + const failedSearches = + round.failedSearches ?? round.queries.length - round.successfulSearches; lines.push(`### Round ${round.round}`); lines.push( `Queries: ${round.queries.map((q) => `"${q.query}" [${q.angle}]`).join(", ")}`, diff --git a/src/research.ts b/src/research.ts index 5f8e474..74ff0eb 100644 --- a/src/research.ts +++ b/src/research.ts @@ -22,7 +22,7 @@ import type { ResearchReport, } from "./types"; import type { SynthesisResult } from "./report"; -import { searchWeb } from "./firecrawl"; +import { searchWeb, isNearDuplicateTitle } from "./firecrawl"; import { generateQueries, generateFollowUpQueries, @@ -221,6 +221,7 @@ export async function runDeepResearch( // Run searches in parallel using round-robin bounded concurrency. // Each mapper call runs independently; failures are caught per-query. + // Results keep their originating query index for later grouping. const searchResultsArrays: (EnrichedSearchResult[] | null)[] = await boundedConcurrency( currentQueries, @@ -252,23 +253,71 @@ export async function runDeepResearch( }, ); - // Flatten results, filtering out nulls (failed searches) - const searchResults: EnrichedSearchResult[] = searchResultsArrays - .filter((r): r is EnrichedSearchResult[] => r !== null) - .flat(); + const successfulSearches = searchResultsArrays.filter( + (r): r is EnrichedSearchResult[] => r !== null, + ).length; + const failedSearches = currentQueries.length - successfulSearches; totalSearches += currentQueries.length; - // Deduplicate results by URL (prefer higher authority) - const seen = new Map(); - for (const r of searchResults) { - const existing = seen.get(r.url); - if (!existing || r.authorityScore > existing.authorityScore) { - seen.set(r.url, r); + // Track which URLs were independently surfaced by MULTIPLE different + // queries. This is the corroboration signal: a source found by + // several independent searches is stronger evidence than one found + // by a single query. + const urlQueryCounts = new Map(); + searchResultsArrays.forEach((results, queryIndex) => { + if (!results) return; + const queryUrls = new Set(results.map((r) => r.url)); + for (const url of queryUrls) { + urlQueryCounts.set(url, (urlQueryCounts.get(url) ?? 0) + 1); + } + // (queryIndex is unused beyond the closure; kept for clarity) + void queryIndex; + }); + + // ── Per-query result collection ────────────────────────────────── + // Deduplicate within each query's results by URL (prefer higher + // authority) AND by near-identical title (catches syndicated copies + // of the same article under different URLs). + const resultsByQuery: EnrichedSearchResult[][] = currentQueries.map( + () => [], + ); + + searchResultsArrays.forEach((results, queryIndex) => { + if (!results) return; + const seenUrls = new Set(); + const seenTitles: string[] = []; + for (const r of results) { + if (seenUrls.has(r.url)) continue; + seenUrls.add(r.url); + + // Skip syndicated duplicates (same article, different URL) + if ( + r.title && + seenTitles.some((t) => isNearDuplicateTitle(t, r.title)) + ) { + continue; + } + if (r.title) seenTitles.push(r.title); + resultsByQuery[queryIndex].push(r); + } + }); + + // Global URL dedup across queries: a URL found by multiple queries + // stays attached to the FIRST query that surfaced it (most likely + // the most relevant one). + const globalSeen = new Set(); + for (const list of resultsByQuery) { + for (let i = list.length - 1; i >= 0; i--) { + if (globalSeen.has(list[i].url)) { + list.splice(i, 1); + } else { + globalSeen.add(list[i].url); + } } } - const uniqueResults = Array.from(seen.values()); + const uniqueResults = resultsByQuery.flat(); totalPages += uniqueResults.length; // ── Analyze phase (parallel with round-robin) ────────────────── @@ -283,23 +332,18 @@ export async function runDeepResearch( if (signal?.aborted) throw new Error("Research cancelled"); - // Build query-result pairs for parallel analysis + // Build query-result pairs for parallel analysis. + // Each query is analyzed with the results IT actually produced, + // so findings stay coherent with the query's intent and angle. const analysisTasks: Array<{ query: (typeof currentQueries)[number]; - results: typeof uniqueResults; + results: EnrichedSearchResult[]; index: number; }> = []; - const resultsPerQuery = Math.ceil( - uniqueResults.length / currentQueries.length, - ); - for (let i = 0; i < currentQueries.length; i++) { - const startIdx = i * resultsPerQuery; - const endIdx = Math.min(startIdx + resultsPerQuery, uniqueResults.length); - const queryResults = uniqueResults.slice(startIdx, endIdx); - - if (queryResults.length === 0) continue; + const queryResults = resultsByQuery[i]; + if (!queryResults || queryResults.length === 0) continue; analysisTasks.push({ query: currentQueries[i], @@ -318,7 +362,8 @@ export async function runDeepResearch( round, totalRounds: config.depth, message: `Analyzing: "${task.query.query.slice(0, 40)}..."`, - fraction: 0.6 + (task.index / currentQueries.length) * 0.2, + fraction: + 0.6 + (task.index / Math.max(analysisTasks.length, 1)) * 0.2, }); try { @@ -327,6 +372,7 @@ export async function runDeepResearch( task.results, ctx.cwd, signal, + task.query.angle, ); } catch { // Analysis failure shouldn't crash the round @@ -339,11 +385,15 @@ export async function runDeepResearch( const allFindings: ResearchRound["findings"] = findingsArrays.flat(); // ── Corroboration pass ──────────────────────────────────────── - // Cross-reference findings to compute corroboration scores - const corroboratedFindings = computeCorroboration(allFindings); + // Cross-reference findings to compute corroboration scores. + // Corroboration = fraction of a finding's sources that were + // independently surfaced by multiple different search queries. + const corroboratedFindings = computeCorroboration( + allFindings, + urlQueryCounts, + ); // Record this round - const successfulSearches = currentQueries.length; const followUpTopics = corroboratedFindings .filter( (f: Finding) => @@ -358,6 +408,7 @@ export async function runDeepResearch( findings: corroboratedFindings, followUpTopics, successfulSearches, + failedSearches, }); // ── Adaptive depth: check for saturation ────────────────────── diff --git a/src/types.ts b/src/types.ts index 5b9d4cc..e880d31 100644 --- a/src/types.ts +++ b/src/types.ts @@ -35,6 +35,10 @@ export interface Finding { sources: string[]; keyQuotes: string[]; confidence: "high" | "medium" | "low"; + /** The search query this finding was extracted under (provenance) */ + query?: string; + /** The research angle of the originating query (provenance) */ + angle?: string; /** 0.0 – 1.0: how many independent sources support this finding */ corroborationScore?: number; /** Authority score of the best source supporting this finding */ @@ -68,8 +72,10 @@ export interface ResearchRound { findings: Finding[]; /** Any follow-up questions/angles the analysis suggests */ followUpTopics: string[]; - /** Number of sources that actually returned data (non-empty) */ + /** Number of search queries that actually returned data (non-empty) */ successfulSearches: number; + /** Number of search queries that failed entirely */ + failedSearches: number; } /** Target audience expertise level */