/** * deep-research — Multi-round deep web research powered by Firecrawl * * Registers: * - `deep_research` tool — callable by the LLM to conduct deep research * - `/deep-research` command — interactive session invocation * * Architecture: * Each research round generates queries, searches in parallel via * Firecrawl, analyzes results with agent sessions, then generates * follow-up queries. A final synthesis step produces the report. * * Patterns borrowed from: * - firecrawl.ts extension (direct Firecrawl HTTP calls) * - ralpi executor (agent sessions, widget updates, progress UX) * - subagent extension (structured tool rendering) */ import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; import { Box, Text } from "@earendil-works/pi-tui"; import { runDeepResearch, type ResearchProgress } from "./src/research"; import { isFirecrawlReachable } from "./src/firecrawl"; import type { ResearchConfig, ResearchReport } from "./src/types"; /* ── Constants ────────────────────────────────────────────────────── */ const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]; const PHASE_ICONS: Record = { generating_queries: "🔍", searching: "🌐", analyzing: "📊", synthesizing: "📝", complete: "✅", }; /* ── Helpers ──────────────────────────────────────────────────────── */ function formatDuration(ms: number): string { const seconds = Math.floor(ms / 1000); const minutes = Math.floor(seconds / 60); if (minutes > 0) return `${minutes}m ${seconds % 60}s`; return `${seconds}s`; } function truncate(s: string, max: number): string { if (s.length <= max) return s; return s.slice(0, max - 3) + "..."; } /* ── Tool Definition ──────────────────────────────────────────────── */ const DeepResearchParams = Type.Object({ question: Type.String({ description: "The research question to investigate", }), depth: Type.Optional( Type.Integer({ description: "Number of research rounds (1-3). Each round uses findings from the previous to generate deeper follow-up queries. Default: 2", minimum: 1, maximum: 3, default: 2, }), ), breadth: Type.Optional( Type.Integer({ description: "Number of search queries per round (1-5). More queries = broader coverage. Default: 3", minimum: 1, maximum: 5, default: 3, }), ), format: Type.Optional( Type.Union([Type.Literal("markdown"), Type.Literal("structured")], { description: 'Output format for the research report. "markdown" for prose, "structured" for detailed sections. Default: "markdown"', default: "markdown", }), ), details: Type.Optional( Type.Object({ showRoundDetails: Type.Optional( Type.Boolean({ description: "Include per-round search details in the output. Default: false", }), ), }), ), }); interface ResearchDetails { rounds: Array<{ round: number; queries: string[]; findingsCount: number; resultsCount: number; }>; totalSearches: number; totalPagesScraped: number; durationMs: number; } /* ── Extension Entry ───────────────────────────────────────────────── */ export default function (pi: ExtensionAPI) { pi.registerTool({ name: "deep_research", label: "Deep Research", description: [ "Conduct multi-round deep web research on any topic using Firecrawl.", "Generates diverse search queries, searches the web in parallel, analyzes results, and produces a comprehensive report.", "Supports iterative refinement: each round builds on findings from the previous one.", "Parameters: question (required), depth (1-3, default 2), breadth (1-5, default 3), format (markdown|structured).", ].join(" "), promptSnippet: "deep_research — multi-round deep web research via Firecrawl with iterative query refinement", promptGuidelines: [ "Use deep_research for complex, multi-faceted questions that benefit from multiple search angles and iterative refinement.", "The tool handles query generation, web search, result analysis, and report synthesis automatically.", "For simple fact-finding questions, use firecrawl_search directly instead.", ], parameters: DeepResearchParams, async execute( _toolCallId: string, params: { question: string; depth?: number; breadth?: number; format?: "markdown" | "structured"; details?: { showRoundDetails?: boolean }; }, signal: AbortSignal | undefined, onUpdate: ((partial: any) => void) | undefined, ctx: any, ) { const config: ResearchConfig = { question: params.question, depth: params.depth ?? 2, breadth: params.breadth ?? 3, format: params.format ?? "markdown", }; // Use provided signals const abortSignal = signal; // Wire progress updates to both the widget and onUpdate let spinnerIdx = 0; const spinnerTimer = setInterval(() => { spinnerIdx = (spinnerIdx + 1) % SPINNER_FRAMES.length; }, 100); let researchResult: ResearchReport | null = null; let lastError: string | null = null; const onProgress: ResearchProgress = (update) => { const icon = PHASE_ICONS[update.phase] ?? ""; const spinner = SPINNER_FRAMES[spinnerIdx]; const roundInfo = update.round && update.totalRounds ? ` Round ${update.round}/${update.totalRounds}` : ""; // Update widget const lines: string[] = [ `${spinner} ${icon} ${truncate(update.message, 80)}${roundInfo}`, ]; if (update.detail) { lines.push(` ${truncate(update.detail, 76)}`); } if (update.fraction !== undefined) { const barLen = 15; const filled = Math.round(barLen * update.fraction); const bar = "█".repeat(filled) + "░".repeat(barLen - filled); lines.push(` ${bar}`); } ctx.ui.setWidget("deep-research", lines); // Stream partial results via onUpdate if (onUpdate) { const partialText = lines.join("\n"); onUpdate({ content: [{ type: "text", text: partialText }], details: { phase: update.phase, round: update.round, message: update.message, fraction: update.fraction, }, }); } }; try { // Initial status ctx.ui.setStatus( "deep-research", `🌐 Researching: ${truncate(config.question, 40)}`, ); onProgress({ phase: "generating_queries", message: "Starting deep research...", fraction: 0, }); researchResult = await runDeepResearch( config, ctx, onProgress, abortSignal, ); // ── Build the tool result ────────────────────────────────── const details: ResearchDetails = { rounds: researchResult.rounds.map((r) => ({ round: r.round, queries: r.queries.map((q) => q.query), findingsCount: r.findings.length, resultsCount: r.results.length, })), totalSearches: researchResult.totalSearches, totalPagesScraped: researchResult.totalPagesScraped, durationMs: researchResult.durationMs, }; const showRoundDetails = params.details?.showRoundDetails ?? false; let output = researchResult.finalReport; if (showRoundDetails) { output += `\n\n---\n\n## Research Methodology\n\n`; for (const round of researchResult.rounds) { output += `### Round ${round.round}\n\n`; output += `**Queries:**\n`; for (const q of round.queries) { output += `- "${q.query}" (${q.angle}) — ${q.rationale}\n`; } output += `\n**Results scraped:** ${round.results.length}\n`; output += `**Findings extracted:** ${round.findings.length}\n\n`; } output += `**Total searches:** ${researchResult.totalSearches}\n`; output += `**Total pages scraped:** ${researchResult.totalPagesScraped}\n`; output += `**Duration:** ${formatDuration(researchResult.durationMs)}\n`; } // Clean up widget clearInterval(spinnerTimer); ctx.ui.setWidget("deep-research", undefined); ctx.ui.setStatus("deep-research", undefined); return { content: [{ type: "text", text: output }], details, }; } catch (error) { clearInterval(spinnerTimer); ctx.ui.setWidget("deep-research", undefined); ctx.ui.setStatus("deep-research", undefined); lastError = error instanceof Error ? error.message : String(error); return { content: [ { type: "text", text: `Research failed: ${lastError}`, }, ], details: { error: lastError, phase: researchResult ? `completed ${researchResult.rounds.length} rounds` : "preparation", }, isError: true, }; } }, // ── TUI: Render the tool call (collapsed view) ────────────────── renderCall( args: { question: string; depth?: number; breadth?: number; format?: string; }, theme: any, _context: any, ) { const question = truncate(args.question ?? "?", 70); const depth = args.depth ?? 2; const breadth = args.breadth ?? 3; const format = args.format ?? "markdown"; const text = theme.fg("toolTitle", theme.bold("deep_research ")) + theme.fg("accent", `"${question}"`) + theme.fg("muted", ` [depth:${depth} breadth:${breadth} ${format}]`); return new Text(text, 0, 0); }, // ── TUI: Render the tool result (expanded/collapsed) ───────────── renderResult( result: any, { expanded }: { expanded: boolean }, theme: any, _context: any, ) { const details = result.details as ResearchDetails | undefined; if (!details) { const text = result.content?.[0]?.text ?? "(no output)"; return new Text(text, 0, 0); } const container = new Box(); // ── Collapsed view ──────────────────────────────────────────── if (!expanded) { const totalRounds = details.rounds.length; const totalFindings = details.rounds.reduce( (s, r) => s + r.findingsCount, 0, ); const duration = formatDuration(details.durationMs); let text = ""; text += theme.fg("success", "✓ ") + theme.fg("toolTitle", theme.bold("deep research")); text += theme.fg( "muted", ` — ${totalRounds} rounds, ${totalFindings} findings`, ); text += theme.fg("dim", ` (${duration})`); text += "\n"; for (const round of details.rounds) { const icon = round.findingsCount > 0 ? theme.fg("success", "✓") : theme.fg("muted", "·"); text += ` ${icon} ${theme.fg("accent", `Round ${round.round}:`)} `; text += theme.fg( "dim", `${round.queries.length} queries, ${round.resultsCount} pages, ${round.findingsCount} findings`, ); text += "\n"; } text += theme.fg("muted", "(Ctrl+O to expand)"); container.addChild(new Text(text, 0, 0)); return container; } // ── Expanded view ───────────────────────────────────────────── const headerText = theme.fg("toolTitle", theme.bold("Deep Research Results")) + "\n" + theme.fg("dim", `Duration: ${formatDuration(details.durationMs)} | `) + theme.fg("dim", `Searches: ${details.totalSearches} | `) + theme.fg("dim", `Pages scraped: ${details.totalPagesScraped}`); container.addChild(new Text(headerText, 0, 0)); for (const round of details.rounds) { container.addChild(new Text("", 0, 0)); // Spacer const roundHeader = `Round ${round.round}`; container.addChild( new Text(theme.fg("toolTitle", theme.bold(roundHeader)), 0, 0), ); container.addChild( new Text( theme.fg( "dim", `${round.queries.length} queries → ${round.resultsCount} pages → ${round.findingsCount} findings`, ), 0, 0, ), ); for (const q of round.queries) { container.addChild( new Text( theme.fg("muted", " · ") + theme.fg("accent", truncate(q, 70)), 0, 0, ), ); } } return container; }, }); // ── Command ─────────────────────────────────────────────────────── pi.registerCommand("deep-research", { description: "Conduct multi-round deep web research on any topic via Firecrawl. Usage: /deep-research ", handler: async (args: string, ctx: ExtensionCommandContext) => { if (!args || args.trim().length === 0) { ctx.ui.notify( "Usage: /deep-research ", "error", ); return; } // Ask about depth/breadth const depthStr = await ctx.ui.select("Research depth?", [ "1 round (quick survey)", "2 rounds (standard)", "3 rounds (deep dive)", ]); const depth = depthStr?.startsWith("1") ? 1 : depthStr?.startsWith("3") ? 3 : 2; const breadthStr = await ctx.ui.select("Research breadth?", [ "1 query/round (narrow)", "3 queries/round (balanced)", "5 queries/round (broad)", ]); const breadth = breadthStr?.startsWith("1") ? 1 : breadthStr?.startsWith("5") ? 5 : 3; // Create a promise-based interaction ctx.ui.setStatus( "deep-research", `🌐 Researching: ${truncate(args, 40)}`, ); const config: ResearchConfig = { question: args, depth, breadth, format: "markdown", }; let spinnerIdx = 0; const spinnerTimer = setInterval(() => { spinnerIdx = (spinnerIdx + 1) % SPINNER_FRAMES.length; }, 100); try { const onProgress: ResearchProgress = (update) => { const icon = PHASE_ICONS[update.phase] ?? ""; const spinner = SPINNER_FRAMES[spinnerIdx]; const lines: string[] = [ `${spinner} ${icon} ${truncate(update.message, 80)}`, ]; if (update.detail) { lines.push(` ${truncate(update.detail, 76)}`); } if (update.fraction !== undefined) { const barLen = 15; const filled = Math.round(barLen * update.fraction); const bar = "█".repeat(filled) + "░".repeat(barLen - filled); lines.push(` ${bar}`); } ctx.ui.setWidget("deep-research", lines); }; const report = await runDeepResearch(config, ctx, onProgress); clearInterval(spinnerTimer); ctx.ui.setWidget("deep-research", undefined); ctx.ui.setStatus("deep-research", undefined); // Show notification ctx.ui.notify( `Research complete: ${report.rounds.length} rounds, ${report.totalSearches} searches, ${report.totalPagesScraped} pages in ${formatDuration(report.durationMs)}`, "info", ); // Send the report as a user message pi.sendUserMessage( `## Deep Research: ${args}\n\n${report.finalReport}\n\n---\n*${report.rounds.length} rounds · ${report.totalSearches} searches · ${report.totalPagesScraped} pages · ${formatDuration(report.durationMs)}*`, ); } catch (error) { clearInterval(spinnerTimer); ctx.ui.setWidget("deep-research", undefined); ctx.ui.setStatus("deep-research", undefined); ctx.ui.notify( `Research failed: error instanceof Error ? error.message : String(error)`, "error", ); } }, }); // ── Startup check ───────────────────────────────────────────────── pi.on("session_start", async (_event: unknown, ctx: ExtensionContext) => { const reachable = await isFirecrawlReachable(); if (!reachable) { ctx.ui.notify( "Deep Research: Firecrawl endpoint unreachable — searches will fail. Check FIRECRAWL_BASE_URL in settings.json or env.", "warning", ); } }); }