From 88abadfabf1c7fde2f9097045f0b838dda35ab01 Mon Sep 17 00:00:00 2001 From: Michael Freno Date: Mon, 10 Aug 2026 09:46:09 -0400 Subject: [PATCH] initial import: @mikefreno/omp-deepi-research (omp port) --- .gitignore | 5 + LICENSE | 21 ++ README.md | 115 ++++++++ bun.lock | 412 +++++++++++++++++++++++++++ index.ts | 590 ++++++++++++++++++++++++++++++++++++++ package.json | 43 +++ scripts/run-harness.ts | 115 ++++++++ src/agent.ts | 150 ++++++++++ src/firecrawl.ts | 525 ++++++++++++++++++++++++++++++++++ src/queries.ts | 625 +++++++++++++++++++++++++++++++++++++++++ src/report.ts | 530 ++++++++++++++++++++++++++++++++++ src/research.ts | 468 ++++++++++++++++++++++++++++++ src/types.ts | 106 +++++++ tsconfig.json | 16 ++ 14 files changed, 3721 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 bun.lock create mode 100644 index.ts create mode 100644 package.json create mode 100644 scripts/run-harness.ts create mode 100644 src/agent.ts create mode 100644 src/firecrawl.ts create mode 100644 src/queries.ts create mode 100644 src/report.ts create mode 100644 src/research.ts create mode 100644 src/types.ts create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5afeedc --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +dist/ +.pi-lens/ +AGENTS.md +package-lock.json diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8fa6dd9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Michael Freno + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d48224 --- /dev/null +++ b/README.md @@ -0,0 +1,115 @@ +# Deep Research + +Multi-round deep web research powered by Firecrawl with iterative query refinement. + +Deep Research is a local omp extension under `~/.omp/agent/extensions/deepi-research/`. +Omp loads it via `omp.extensions` in `package.json` (entry `./index.ts`). + +## Features + +- **Multi-round iteration**: Each round generates follow-up queries based on previous findings (depth 1-3) +- **Parallel query expansion**: Multiple diverse search queries per round (breadth 1-5) covering technical, practical, comparative, critical, and forward-looking angles +- **Sub-question decomposition**: Broad questions are broken into focused sub-topics before query generation (depth > 1) +- **Round-robin parallel execution**: Searches and analyses run concurrently within each round using bounded-concurrency worker pools, dramatically reducing total research time +- **LLM-driven analysis**: Each query's results are analyzed by its own agent session (per-query provenance) to extract structured findings with confidence ratings +- **Source authority scoring**: Every source is scored by domain authority; low-quality SEO domains are penalized with a hard floor; findings are ranked by authority × confidence before synthesis +- **Cross-query corroboration**: A finding is corroborated only when its sources were independently surfaced by multiple different search queries +- **Citation integrity**: References are rebuilt from the authoritative bibliography (never the LLM's), and hallucinated inline citation numbers are stripped +- **Near-duplicate detection**: Syndicated copies of the same article are removed by title similarity, and duplicate findings across rounds are merged +- **Automatic deduplication**: Search results are deduplicated by URL across all queries +- **Robust LLM output parsing**: JSON output with code fences, prose prefixes, or trailing commas is parsed reliably +- **Graceful degradation**: Individual search or analysis failures don't crash the full research — partial results are preserved, with retry-with-backoff for transient Firecrawl errors +- **Progress streaming**: Real-time progress widget with spinner, phase indicators, and progress bar +- **Abort support**: Research can be cancelled mid-flight via `AbortSignal` +- **Rich TUI rendering**: Compact collapsed view and detailed expanded view in the terminal UI +- **Fallback resilience**: Built-in fallback query generation and report synthesis when LLM calls fail + +## Usage + +### Tool (LLM-callable) + +Registers the `deep_research` tool for AI agent use: + +``` +deep_research — multi-round deep web research via Firecrawl with iterative query refinement +``` + +Parameters: + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `question` | string | — | The research question to investigate | +| `depth` | integer (1-3) | 2 | Number of research rounds | +| `breadth` | integer (1-5) | 3 | Search queries per round | +| `format` | "markdown" \| "structured" | "markdown" | Output format for the report | +| `audience` | "general" \| "expert" \| "executive" | "general" | Tone and depth for the report audience | +| `details.showRoundDetails` | boolean | false | Include per-round search metadata (incl. failed searches) in output | + +### Command (interactive) + +``` +/deepi-research +``` + +Prompts for depth (1-3 rounds) and breadth (1-5 queries) interactively, then runs the research and sends the final report as a user message. + +### Recommended usage + +- Use `deep_research` for complex, multi-faceted questions that benefit from multiple search angles and iterative refinement. +- The tool handles query generation, web search, result analysis, and report synthesis automatically. +- For simple fact-finding questions, use `firecrawl_search` directly instead. + +## Architecture + +``` +Research Flow: + + Question + ↓ + ┌─ Round 1 ───────────────────────────┐ + │ LLM → generate queries (N angles) │ + │ Firecrawl → search each query │ + │ LLM → analyze results → findings │ + └──────────────┬───────────────────────┘ + ↓ (follow-up queries) + ┌─ Round 2 ───────────────────────────┐ + │ LLM → identify knowledge gaps │ + │ Firecrawl → search follow-ups │ + │ LLM → analyze → new findings │ + └──────────────┬───────────────────────┘ + ↓ (iterate depth times) + ┌─ Synthesis ─────────────────────────┐ + │ LLM → synthesize all findings │ + │ → comprehensive research report │ + └─────────────────────────────────────┘ +``` + +## Configuration + +Deep Research reads Firecrawl configuration from omp's config.yml files, with the following resolution order (later wins): + +1. Environment variables (`FIRECRAWL_BASE_URL`, `FIRECRAWL_API_KEY`) +2. Global config (`$agentDir/config.yml`) → `firecrawl.*` +3. Project config (`.omp/config.yml`) → `firecrawl.*` +4. Default `http://localhost:3002` (if nothing else sets baseUrl) + +The agent directory (`$agentDir`) defaults to `~/.omp/agent`. + +**Global config** (`~/.omp/agent/config.yml`): + +```yaml +firecrawl: + baseUrl: http://localhost:3002 +``` + +**Project config** (`.omp/config.yml` — overrides global): + +```yaml +firecrawl: + baseUrl: https://firecrawl.team.internal + apiKey: your-api-key +``` + +### Session startup check + +On `session_start`, the extension checks whether the Firecrawl endpoint is reachable. If not, it shows a warning notification so you know searches will fail before you try to use it. diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..cc79b60 --- /dev/null +++ b/bun.lock @@ -0,0 +1,412 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@mikefreno/deepi-research", + "dependencies": { + "yaml": "^2.4.0", + }, + "devDependencies": { + "@oh-my-pi/pi-coding-agent": "17.2.12", + "@oh-my-pi/pi-tui": "17.2.12", + "@types/node": "^20.0.0", + "typebox": "^1.1.0", + "typescript": "^5.3.0", + }, + }, + }, + "packages": { + "@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], + + "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], + + "@babel/parser": ["@babel/parser@7.29.8", "", { "dependencies": { "@babel/types": "^7.29.8" }, "bin": "./bin/babel-parser.js" }, "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA=="], + + "@babel/types": ["@babel/types@7.29.8", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg=="], + + "@bufbuild/protobuf": ["@bufbuild/protobuf@2.13.0", "", {}, "sha512-acq7c49vxfm1ggJ95P70TX7ABDM0vxr1SYD3BB0o0jnBLB4OAqeHyKuN+cD3w80gXEDQ2zxHpR6CUeA+O/aU9g=="], + + "@emnapi/runtime": ["@emnapi/runtime@1.11.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA=="], + + "@huggingface/jinja": ["@huggingface/jinja@0.5.9", "", {}, "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw=="], + + "@huggingface/tokenizers": ["@huggingface/tokenizers@0.1.3", "", {}, "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA=="], + + "@huggingface/transformers": ["@huggingface/transformers@4.2.0", "", { "dependencies": { "@huggingface/jinja": "^0.5.6", "@huggingface/tokenizers": "^0.1.3", "onnxruntime-node": "1.24.3", "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", "sharp": "^0.34.5" } }, "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ=="], + + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], + + "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], + + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + + "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + + "@kurkle/color": ["@kurkle/color@0.3.4", "", {}, "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w=="], + + "@oh-my-pi/hashline": ["@oh-my-pi/hashline@17.2.12", "", { "dependencies": { "@oh-my-pi/pi-natives": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12" } }, "sha512-OdqMAojK8iUZynyNAP2VnbB+EHOHM7ISDSM3f9xHh/sDNSPDi15Xwd53AIdJy+EhYoCyglDIqO9/seHjA6veQQ=="], + + "@oh-my-pi/omp-stats": ["@oh-my-pi/omp-stats@17.2.12", "", { "dependencies": { "@oh-my-pi/pi-ai": "17.2.12", "@oh-my-pi/pi-catalog": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12", "@tailwindcss/node": "^4.3.2", "chart.js": "^4.5.1", "lucide-react": "^1.24.0", "react": "19.2.7", "react-chartjs-2": "^5.3.1", "react-dom": "19.2.7", "tailwindcss": "^4.3.2" }, "bin": { "omp-stats": "src/index.ts" } }, "sha512-BOyDNq8Hj/CJVc6Wntdo+1yaG/b8CqLcxsBnHefLoC7vO96YoFjt4S7GOpp3cd+ysiuxHnmrWn30kEtwR9fhjQ=="], + + "@oh-my-pi/omptype": ["@oh-my-pi/omptype@17.2.12", "", {}, "sha512-Y27VMnPbcUEOpK1DgYuvNIaVnGxEmOcEP2bYPfuAWkx2vc4PKXL0NCRzGLwYMjhwjjd9O/gn8lywZfxDCBytjA=="], + + "@oh-my-pi/pi-agent-core": ["@oh-my-pi/pi-agent-core@17.2.12", "", { "dependencies": { "@oh-my-pi/pi-ai": "17.2.12", "@oh-my-pi/pi-catalog": "17.2.12", "@oh-my-pi/pi-natives": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12", "@oh-my-pi/pi-wire": "17.2.12", "@oh-my-pi/snapcompact": "17.2.12", "@opentelemetry/api": "^1.9.1" } }, "sha512-VIJnZCQyshiZoiWpvhoNVHhUQmptLG41h7XPg2IZfEdDNdDpV26JuZdDMgBr0hIz1L/RfTq8MiRcRuAcasvkOg=="], + + "@oh-my-pi/pi-ai": ["@oh-my-pi/pi-ai@17.2.12", "", { "dependencies": { "@bufbuild/protobuf": "^2.12.1", "@oh-my-pi/omptype": "17.2.12", "@oh-my-pi/pi-catalog": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12", "@oh-my-pi/pi-wire": "17.2.12" } }, "sha512-S6LswgyLgQdE5zXADf6NGwrYn1OCYp1WiKxyN6f/1RRv68vbd3y4YSoidzeOYs3kPS/cIiv3dw3LHaCrwpam/Q=="], + + "@oh-my-pi/pi-catalog": ["@oh-my-pi/pi-catalog@17.2.12", "", { "dependencies": { "@bufbuild/protobuf": "^2.12.1", "@oh-my-pi/omptype": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12" } }, "sha512-6VycCR0ShSzbVfOa7cdeqVtxHL2IPpJXFePKaQPhQ8bOTiyE/TaUm1uIa1i4doqPTz/k6vUEs+a57cZwoUP97A=="], + + "@oh-my-pi/pi-coding-agent": ["@oh-my-pi/pi-coding-agent@17.2.12", "", { "dependencies": { "@babel/parser": "^7.29.7", "@oh-my-pi/hashline": "17.2.12", "@oh-my-pi/omp-stats": "17.2.12", "@oh-my-pi/omptype": "17.2.12", "@oh-my-pi/pi-agent-core": "17.2.12", "@oh-my-pi/pi-ai": "17.2.12", "@oh-my-pi/pi-catalog": "17.2.12", "@oh-my-pi/pi-mnemopi": "17.2.12", "@oh-my-pi/pi-natives": "17.2.12", "@oh-my-pi/pi-tui": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12", "@oh-my-pi/pi-wire": "17.2.12", "@oh-my-pi/snapcompact": "17.2.12", "@opentelemetry/api": "^1.9.1", "@opentelemetry/api-logs": "^0.220.0", "@opentelemetry/context-async-hooks": "^2.9.0", "@opentelemetry/exporter-logs-otlp-proto": "^0.220.0", "@opentelemetry/exporter-metrics-otlp-proto": "^0.220.0", "@opentelemetry/exporter-trace-otlp-proto": "^0.220.0", "@opentelemetry/resources": "^2.9.0", "@opentelemetry/sdk-logs": "^0.220.0", "@opentelemetry/sdk-metrics": "^2.9.0", "@opentelemetry/sdk-trace-base": "^2.9.0", "@opentelemetry/sdk-trace-node": "^2.9.0", "mupdf": "^1.28.0", "puppeteer-core": "25.3.0" }, "optionalDependencies": { "@huggingface/transformers": "^4.2.0", "sherpa-onnx-node": "1.13.2" }, "bin": { "omp": "dist/cli.js" } }, "sha512-+q+W4fyNQQ7xAKiN0mmOisWDDtKO0R/ZctTSsKqR4ulN3K1zfQ9HwiTxtg7HJHn5fwCy+X3BmUG72FatNUN8IA=="], + + "@oh-my-pi/pi-mnemopi": ["@oh-my-pi/pi-mnemopi@17.2.12", "", { "dependencies": { "@oh-my-pi/pi-ai": "17.2.12", "@oh-my-pi/pi-catalog": "17.2.12", "@oh-my-pi/pi-natives": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12" }, "peerDependencies": { "fastembed": "2.1.0", "onnxruntime-node": "1.21.0" }, "optionalPeers": ["fastembed", "onnxruntime-node"], "bin": { "mnemopi": "src/cli.ts" } }, "sha512-lKjEexuFC/piaNnb3MzJKu8rolXwkjfv0M90UBUiNPFxKbpKHWU5np5aUAOYkT+fwKoQPQ7Tr95mJx0l32S0OQ=="], + + "@oh-my-pi/pi-natives": ["@oh-my-pi/pi-natives@17.2.12", "", { "optionalDependencies": { "@oh-my-pi/pi-natives-darwin-arm64": "17.2.12", "@oh-my-pi/pi-natives-darwin-x64": "17.2.12", "@oh-my-pi/pi-natives-linux-arm64": "17.2.12", "@oh-my-pi/pi-natives-linux-x64": "17.2.12", "@oh-my-pi/pi-natives-win32-x64": "17.2.12" } }, "sha512-MVZq0UrrA7mk6uMKrgjnAhfrDj+58yEuu0VeVd3JuvneMjcX1duIzOdyqRG13S+/XzGOvGkk164dM6D/CwxteQ=="], + + "@oh-my-pi/pi-natives-darwin-arm64": ["@oh-my-pi/pi-natives-darwin-arm64@17.2.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-5iullmWEWmGRjwgo7Kw6Bub3KOXahkUX3jZfw65bM9LjFRceSwoLTt6Yh0RnXXP8hRzZnhU3b/o73peWBlpLzw=="], + + "@oh-my-pi/pi-natives-darwin-x64": ["@oh-my-pi/pi-natives-darwin-x64@17.2.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-qNc2q02d5etsOdxmO4CNwCcEvgfWS9Ya/fuq2hDKJgp3BFeKQCzYDap0HhC/qkAfj2vpjsTyKc/PsPMMMLGytg=="], + + "@oh-my-pi/pi-natives-linux-arm64": ["@oh-my-pi/pi-natives-linux-arm64@17.2.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-HTc6efuPNoYSzHu7Zk4gl3RtQzWWprVA4x0ep0gq5B7mBS6OwgnbH5otLrXSFeQ2UgRyV/nddz1O02ek4Vg8MA=="], + + "@oh-my-pi/pi-natives-linux-x64": ["@oh-my-pi/pi-natives-linux-x64@17.2.12", "", { "os": "linux", "cpu": "x64" }, "sha512-jg9xzfqpcBDGRVnW+98z/eQPIfcnOYujd1bLW0PnJh6A0ykSqjCvt9PpN0wrlqr70WazOOURqoodN2eibIyOOQ=="], + + "@oh-my-pi/pi-natives-win32-x64": ["@oh-my-pi/pi-natives-win32-x64@17.2.12", "", { "os": "win32", "cpu": "x64" }, "sha512-Jb/EF7Ug7SX/YYJofgKR5Kvw0I+AqGpEfmI5inIsvFzygqlA4QgMETbAR1s08sK+AYZqPqRorBpbGNas85XFig=="], + + "@oh-my-pi/pi-tui": ["@oh-my-pi/pi-tui@17.2.12", "", { "dependencies": { "@oh-my-pi/pi-natives": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12" } }, "sha512-X4IKQG3alzw3ogUewxoPBaZImReEBF7P2Xu5xN5E47lfMHR0eNJ6cQnQmk7sHjfNEAclKCJDYqwzgOfRdh9GEA=="], + + "@oh-my-pi/pi-utils": ["@oh-my-pi/pi-utils@17.2.12", "", { "dependencies": { "@oh-my-pi/pi-natives": "17.2.12" } }, "sha512-iYNV2y6RW9tzYaBa7hX5pzb23+zzNLF/Bs4bK7LAO2Qc7OZl5gw1lp9YOUX4/VZOO7A20KrXnZNwZVCnpPKy1A=="], + + "@oh-my-pi/pi-wire": ["@oh-my-pi/pi-wire@17.2.12", "", {}, "sha512-UdQ0VP3gExd+jXgy8epCSZ+PTW9pkFu8FoomywmqEPj/JKjt1SqZhIiN8N51HkQCC5WJryT5AedzL/Qr6OID8w=="], + + "@oh-my-pi/snapcompact": ["@oh-my-pi/snapcompact@17.2.12", "", { "dependencies": { "@oh-my-pi/pi-ai": "17.2.12", "@oh-my-pi/pi-natives": "17.2.12", "@oh-my-pi/pi-utils": "17.2.12", "@oh-my-pi/pi-wire": "17.2.12" } }, "sha512-kWYY/7tgIAnXAiYogC6K1GLU2G26QXTr2vDJGq1mxCMk3/GixoNzrFwzK8w6NmKYdQpO/Sa1SRw9sypYT+v1Bw=="], + + "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], + + "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.220.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w=="], + + "@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.10.0", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-bvyMcgLEkozzSzpEEEo1OMoeQ97bxj6Qs2uN3mPrSdDvObMI1myffD/BPqcLlzZO9//d1SqQA/WPw7Cz2AiqhA=="], + + "@opentelemetry/core": ["@opentelemetry/core@2.9.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-m2nckMT80NnmjTYSPjJQObBJ+8dgkoajEOUbznL8AHZ3T3yHRk2P7gI1PhEBc1+lOnrYE9UWrWHqJDsmqjmNbw=="], + + "@opentelemetry/exporter-logs-otlp-proto": ["@opentelemetry/exporter-logs-otlp-proto@0.220.0", "", { "dependencies": { "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/sdk-logs": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-8LZAxdJ0ENDAFwr4j0oY35mHBltiSzvlhdQAPGiC7p9VnxtuSq4SW1gfBAdW6t6hiQG6OwUl8w7KHaOdJPKHWg=="], + + "@opentelemetry/exporter-metrics-otlp-http": ["@opentelemetry/exporter-metrics-otlp-http@0.220.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-metrics": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-Yqt3RBw/bRVncaE9qIIhk4WfjbAQqXuP9FgAaU+IKPndnLEp/cUqZlSC324+bpmduRz7DoTjig8Ub0PeILWXUA=="], + + "@opentelemetry/exporter-metrics-otlp-proto": ["@opentelemetry/exporter-metrics-otlp-proto@0.220.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/exporter-metrics-otlp-http": "0.220.0", "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-metrics": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lyO+IQBdSvqHN/ZOW/OzrSWemtfD+HgWngn+HBNLhjy0YrCQQTz0OE/kSekH2Pl340dn9DWzhqHdz5Eftr+HLA=="], + + "@opentelemetry/exporter-trace-otlp-proto": ["@opentelemetry/exporter-trace-otlp-proto@0.220.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-exporter-base": "0.220.0", "@opentelemetry/otlp-transformer": "0.220.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-voTAD8XgJxlK7zLkXh8EzMB09zrQr3tyY/BsnDTlDiQU/UdK58MZ63A3mUjdEDrxMjCVmBHU3WQJhRmQe+Dvzg=="], + + "@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.220.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/otlp-transformer": "0.220.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CXYo8UD5Mn9YbgebO2EL4wejtA+gxLmLiu6HCk2KH2BR7XhFN6/6p1UlCb23DYCjeYkndevLHuejCCN1yx4+OQ=="], + + "@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.220.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/sdk-logs": "0.220.0", "@opentelemetry/sdk-metrics": "2.9.0", "@opentelemetry/sdk-trace": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lXGrv7KXZ0gNH9SVNUaa6vv6phVYGvJxfXAlMbzbakiXru75f5MZl8Z7oqiMMQD77riVHJCFlQvbZs/VVN2/4A=="], + + "@opentelemetry/resources": ["@opentelemetry/resources@2.10.0", "", { "dependencies": { "@opentelemetry/core": "2.10.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA=="], + + "@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.220.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.220.0", "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-WywcTkQtv2iNmt+6y5Kcd4rzvx9bLVsBa2Nwcmg01IUaBTkTow3W4d9KE5vNBpEDtb9tp21WcRBY/lANRrApYA=="], + + "@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.10.0", "", { "dependencies": { "@opentelemetry/core": "2.10.0", "@opentelemetry/resources": "2.10.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ=="], + + "@opentelemetry/sdk-trace": ["@opentelemetry/sdk-trace@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-sGA19HvtrrSKYsseHphluH6j3p6Xa3fqc7c7y8f/7mYWejc1lyDFcpSdD1kYa50HCLUeEo4zA5bW0pniaPszuw=="], + + "@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.10.0", "", { "dependencies": { "@opentelemetry/core": "2.10.0", "@opentelemetry/resources": "2.10.0", "@opentelemetry/sdk-trace": "2.10.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-GuYQQT7QD2EeO8lcZLRQzcbOyhqAzL+6WWTKTU9mSUBYBazkEDl+VrQcXQhbB08OWM9anD1aHleVadzulpOaUQ=="], + + "@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.10.0", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.10.0", "@opentelemetry/core": "2.10.0", "@opentelemetry/sdk-trace-base": "2.10.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-GZK/G6oZyBLGlH1pUgeDch7D91KoHd2uotUGIkWCPi9GI5T9X0p4L7nNAMDR1BQjkRYoDqo+ddfVx9t5Uhys+Q=="], + + "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.43.0", "", {}, "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg=="], + + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], + + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], + + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="], + + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="], + + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="], + + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], + + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], + + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], + + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.2", "", {}, "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug=="], + + "@puppeteer/browsers": ["@puppeteer/browsers@3.0.6", "", { "dependencies": { "modern-tar": "^0.7.6", "yargs": "^18.0.0" }, "peerDependencies": { "proxy-agent": ">=8.0.1", "yauzl": "^2.10.0 || ^3.4.0" }, "optionalPeers": ["proxy-agent", "yauzl"], "bin": { "browsers": "lib/main-cli.js" } }, "sha512-B/gKoqlFkzhvzsI6jo9K1cZz9o5ypviVv/xu8CwA4grZzyVwN+XfkT+tu8T1zrauuEXv6VhS2oGX+6NL95WcKA=="], + + "@tailwindcss/node": ["@tailwindcss/node@4.3.3", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.24.1", "jiti": "^2.7.0", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.3.3" } }, "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg=="], + + "@types/node": ["@types/node@20.19.43", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA=="], + + "adm-zip": ["adm-zip@0.5.18", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="], + + "ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], + + "ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="], + + "boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="], + + "chart.js": ["chart.js@4.5.1", "", { "dependencies": { "@kurkle/color": "^0.3.0" } }, "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw=="], + + "chromium-bidi": ["chromium-bidi@16.0.1", "", { "dependencies": { "mitt": "^3.0.1", "zod": "^3.24.1" }, "peerDependencies": { "devtools-protocol": "*" } }, "sha512-J63PGu/9PpeCwLIcKYyzWP6yaVL5pxuBc0shlYCYM8BaAkmlwiQboXO1iNbOgSDbVklEyYFfNEcHD8oOAWacUA=="], + + "cliui": ["cliui@9.0.1", "", { "dependencies": { "string-width": "^7.2.0", "strip-ansi": "^7.1.0", "wrap-ansi": "^9.0.0" } }, "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w=="], + + "define-data-property": ["define-data-property@1.1.4", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="], + + "define-properties": ["define-properties@1.2.1", "", { "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" } }, "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg=="], + + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + + "detect-node": ["detect-node@2.1.0", "", {}, "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g=="], + + "devtools-protocol": ["devtools-protocol@0.0.1638949", "", {}, "sha512-mXwg4Fqnv0WR4iuAT/gYUmctNkjILwXFHyZ+m7Ty1dfr0ezZt2U3gnrrJTfRobJTHoXf+IbuFvFITzLrLFjwJA=="], + + "emoji-regex": ["emoji-regex@10.6.0", "", {}, "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A=="], + + "enhanced-resolve": ["enhanced-resolve@5.24.5", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.3" } }, "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A=="], + + "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], + + "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], + + "es6-error": ["es6-error@4.1.1", "", {}, "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg=="], + + "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], + + "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], + + "flatbuffers": ["flatbuffers@25.9.23", "", {}, "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ=="], + + "get-caller-file": ["get-caller-file@2.0.5", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="], + + "get-east-asian-width": ["get-east-asian-width@1.6.0", "", {}, "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA=="], + + "global-agent": ["global-agent@3.0.0", "", { "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", "matcher": "^3.0.0", "roarr": "^2.15.3", "semver": "^7.3.2", "serialize-error": "^7.0.1" } }, "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q=="], + + "globalthis": ["globalthis@1.0.4", "", { "dependencies": { "define-properties": "^1.2.1", "gopd": "^1.0.1" } }, "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ=="], + + "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], + + "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + + "guid-typescript": ["guid-typescript@1.0.9", "", {}, "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="], + + "has-property-descriptors": ["has-property-descriptors@1.0.2", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="], + + "jiti": ["jiti@2.7.0", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ=="], + + "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], + + "lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], + + "lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="], + + "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.32.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ=="], + + "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.32.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w=="], + + "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.32.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig=="], + + "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.32.0", "", { "os": "linux", "cpu": "arm" }, "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw=="], + + "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.32.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ=="], + + "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.32.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg=="], + + "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.32.0", "", { "os": "linux", "cpu": "x64" }, "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA=="], + + "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.32.0", "", { "os": "linux", "cpu": "x64" }, "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg=="], + + "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.32.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw=="], + + "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.32.0", "", { "os": "win32", "cpu": "x64" }, "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q=="], + + "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + + "lucide-react": ["lucide-react@1.31.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-G8u2eEtoHUnUa9f8lbvqDhCiORMnYLdUEo06EEG9MQvHQrInKcX3Pa2TH39MM5qyzRcWETxB0+aOwAPI1g1kEg=="], + + "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + + "matcher": ["matcher@3.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng=="], + + "mitt": ["mitt@3.0.1", "", {}, "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw=="], + + "modern-tar": ["modern-tar@0.7.7", "", {}, "sha512-t9VmxaqrmANnEOBhpSDI6HD192Ge48k8vmWqQQL7hSFEqHEYwZbbsu49+aKLWZeRvFs3j1pMhXOqqF4kPlvjkQ=="], + + "mupdf": ["mupdf@1.28.0", "", {}, "sha512-ACUnbpECaQ5JLq04pwd89lS+0IGMest5qL5tb08g9TAR7bDtfqflHEkb2Xm3o4rvC/szguLiV+WEbW9kstj8Sg=="], + + "object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="], + + "onnxruntime-common": ["onnxruntime-common@1.24.3", "", {}, "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA=="], + + "onnxruntime-node": ["onnxruntime-node@1.24.3", "", { "dependencies": { "adm-zip": "^0.5.16", "global-agent": "^3.0.0", "onnxruntime-common": "1.24.3" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg=="], + + "onnxruntime-web": ["onnxruntime-web@1.26.0-dev.20260416-b7804b056c", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw=="], + + "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + + "protobufjs": ["protobufjs@7.6.5", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw=="], + + "puppeteer-core": ["puppeteer-core@25.3.0", "", { "dependencies": { "@puppeteer/browsers": "3.0.6", "chromium-bidi": "16.0.1", "devtools-protocol": "0.0.1638949", "typed-query-selector": "^2.12.2", "webdriver-bidi-protocol": "0.4.2", "ws": "^8.21.0" } }, "sha512-fm+wpUr2oigH1PXZvwgATrM2tYWHMDG8ASzTEe9uukCye4X5Ldx1K5BTHPFKITrIWvQQAQ256d1NpbEveBcKjA=="], + + "react": ["react@19.2.7", "", {}, "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ=="], + + "react-chartjs-2": ["react-chartjs-2@5.3.1", "", { "peerDependencies": { "chart.js": "^4.1.1", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-h5IPXKg9EXpjoBzUfyWJvllMjG2mQ4EiuHQFhms/AjUm0XSZHhyRy2xVmLXHKrtcdrPO4mnGqRtYoD0vp95A0A=="], + + "react-dom": ["react-dom@19.2.7", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.7" } }, "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ=="], + + "roarr": ["roarr@2.15.4", "", { "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", "globalthis": "^1.0.1", "json-stringify-safe": "^5.0.1", "semver-compare": "^1.0.0", "sprintf-js": "^1.1.2" } }, "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A=="], + + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], + + "semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + + "semver-compare": ["semver-compare@1.0.0", "", {}, "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow=="], + + "serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], + + "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + + "sherpa-onnx-darwin-arm64": ["sherpa-onnx-darwin-arm64@1.13.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QcYKzyrTzGSx6aKCD6hUODgRS1LetqfG57Z/+i5LCyfMlrgCvDc1lRcl9cdB+TozBsLha9QwLTlI0vmDcf5JKg=="], + + "sherpa-onnx-darwin-x64": ["sherpa-onnx-darwin-x64@1.13.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-6RGeis9K9gV/UQWOgd6Rf3iqXr2/YsBQswxHaCR4hrYkHfEIpHMfFmRWLt6nJJCOWgYW2xFxEd9yzjrafAV/Pw=="], + + "sherpa-onnx-linux-arm64": ["sherpa-onnx-linux-arm64@1.13.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-RMjMRqT82BgTXypNNGmLe6ZFYhc3WEvnAGl3DdkK7qB/kuXwkL3iHhV31wAecbnWPsnEpUoD+8cFovWSBzsCuw=="], + + "sherpa-onnx-linux-x64": ["sherpa-onnx-linux-x64@1.13.4", "", { "os": "linux", "cpu": "x64" }, "sha512-WZh5NCkGPFHHpYSd78iN4OnmxQeSTGyt9uZskH+im/NFHQ7elQ7B0sLzCMeRpvJxiIKvd9C6WxIJ4hYaxClfsQ=="], + + "sherpa-onnx-node": ["sherpa-onnx-node@1.13.2", "", { "optionalDependencies": { "sherpa-onnx-darwin-arm64": "^1.13.2", "sherpa-onnx-darwin-x64": "^1.13.2", "sherpa-onnx-linux-arm64": "^1.13.2", "sherpa-onnx-linux-x64": "^1.13.2", "sherpa-onnx-win-ia32": "^1.13.2", "sherpa-onnx-win-x64": "^1.13.2" } }, "sha512-uIH6SA5Or4pb8HlCYWB3K54XkMtzdef4/tkw1amtIf8GB1tt6hQLpur9p2jSFNfTYRyzZ8XrXofxefXQ0A7EUA=="], + + "sherpa-onnx-win-ia32": ["sherpa-onnx-win-ia32@1.13.4", "", { "os": "win32", "cpu": "ia32" }, "sha512-/JbPjldrfNv+t+uIS3MlkuhfIf5l3FHUGkRC2oRXgjRqOaVmEyP3vLlQ7dTa4J7raG5oB8c3GoPjuSWSqT9GOQ=="], + + "sherpa-onnx-win-x64": ["sherpa-onnx-win-x64@1.13.4", "", { "os": "win32", "cpu": "x64" }, "sha512-R0PWby1VxC14TDZPq7GcfSyXSY6SAFO8Y4JwdCdqouFmeXkZ1L7Is9m98C9KxQ0dN7ZtDzhAmE/43FUs/elXRQ=="], + + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + + "sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], + + "string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], + + "strip-ansi": ["strip-ansi@7.2.0", "", { "dependencies": { "ansi-regex": "^6.2.2" } }, "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w=="], + + "tailwindcss": ["tailwindcss@4.3.3", "", {}, "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ=="], + + "tapable": ["tapable@2.3.3", "", {}, "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], + + "typebox": ["typebox@1.3.11", "", {}, "sha512-tZGKIS02Opbh4EYMmAEVuXl+y3EQJ9ZlkitLZ1CmFCY4ZOqr/xWR9dDSCFmIjNDICGMWkOqMh6WxGTyRXqcSGQ=="], + + "typed-query-selector": ["typed-query-selector@2.12.2", "", {}, "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + + "webdriver-bidi-protocol": ["webdriver-bidi-protocol@0.4.2", "", {}, "sha512-VSV+fzfChirL3e7jay2yUC7B4HQCGtEWEg/MSSQbK+qWbqeGlRLlXTzPpYr3XGUvbpDHumWZBJxgesg4N7dbtA=="], + + "wrap-ansi": ["wrap-ansi@9.0.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww=="], + + "ws": ["ws@8.21.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw=="], + + "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], + + "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], + + "yargs": ["yargs@18.1.0", "", { "dependencies": { "cliui": "^9.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "string-width": "^8.2.1", "y18n": "^5.0.5", "yargs-parser": "^22.0.0" } }, "sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg=="], + + "yargs-parser": ["yargs-parser@22.0.0", "", {}, "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], + + "@opentelemetry/exporter-metrics-otlp-http/@opentelemetry/resources": ["@opentelemetry/resources@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="], + + "@opentelemetry/exporter-metrics-otlp-http/@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q=="], + + "@opentelemetry/exporter-metrics-otlp-proto/@opentelemetry/resources": ["@opentelemetry/resources@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="], + + "@opentelemetry/exporter-metrics-otlp-proto/@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q=="], + + "@opentelemetry/exporter-trace-otlp-proto/@opentelemetry/resources": ["@opentelemetry/resources@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="], + + "@opentelemetry/otlp-transformer/@opentelemetry/resources": ["@opentelemetry/resources@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="], + + "@opentelemetry/otlp-transformer/@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/resources": "2.9.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q=="], + + "@opentelemetry/resources/@opentelemetry/core": ["@opentelemetry/core@2.10.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ=="], + + "@opentelemetry/sdk-logs/@opentelemetry/resources": ["@opentelemetry/resources@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="], + + "@opentelemetry/sdk-metrics/@opentelemetry/core": ["@opentelemetry/core@2.10.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ=="], + + "@opentelemetry/sdk-trace/@opentelemetry/resources": ["@opentelemetry/resources@2.9.0", "", { "dependencies": { "@opentelemetry/core": "2.9.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg=="], + + "@opentelemetry/sdk-trace-base/@opentelemetry/core": ["@opentelemetry/core@2.10.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ=="], + + "@opentelemetry/sdk-trace-base/@opentelemetry/sdk-trace": ["@opentelemetry/sdk-trace@2.10.0", "", { "dependencies": { "@opentelemetry/core": "2.10.0", "@opentelemetry/resources": "2.10.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-MfQGq3GRmTh5fM/y+OjaO0vj6+luCB1XO2gfXCalKCfgKw0eHL++sm75DNweC6ohlp+aFvACqeE0fYayqdRaoQ=="], + + "@opentelemetry/sdk-trace-node/@opentelemetry/core": ["@opentelemetry/core@2.10.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ=="], + + "cliui/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], + + "onnxruntime-web/onnxruntime-common": ["onnxruntime-common@1.24.0-dev.20251116-b39e144322", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="], + + "wrap-ansi/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], + } +} diff --git a/index.ts b/index.ts new file mode 100644 index 0000000..adabbf4 --- /dev/null +++ b/index.ts @@ -0,0 +1,590 @@ +/** + * deep-research — Multi-round deep web research powered by Firecrawl + * + * Registers: + * - `deep_research` tool — callable by the LLM to conduct deep research + * - `/deepi-research` command — interactive session invocation + * + * Architecture: + * Each research round generates queries, searches in parallel via + * Firecrawl, analyzes results with agent sessions, then generates + * follow-up queries. A final synthesis step produces the report. + * + * Patterns borrowed from: + * - firecrawl.ts extension (direct Firecrawl HTTP calls) + * - ralpi executor (agent sessions, widget updates, progress UX) + * - subagent extension (structured tool rendering) + */ +import type { + ExtensionAPI, + ExtensionCommandContext, + ExtensionContext, +} from "@oh-my-pi/pi-coding-agent"; +import type { TSchema } from "@oh-my-pi/pi-ai"; +import { Type } from "typebox"; +import { + Box, + Text, + truncateToWidth, + visibleWidth, +} from "@oh-my-pi/pi-tui"; +import { runDeepResearch, type ResearchProgress } from "./src/research"; +import { isFirecrawlReachable } from "./src/firecrawl"; +import type { ResearchConfig, ResearchReport, Audience } from "./src/types"; + +/* ── Constants ────────────────────────────────────────────────────── */ + +const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]; +const PHASE_ICONS: Record = { + decomposing: "🧩", + generating_queries: "🔍", + searching: "🌐", + analyzing: "📊", + synthesizing: "📝", + complete: "✅", +}; + +type ResearchPhase = Parameters[0]["phase"]; + +/* ── Helpers ──────────────────────────────────────────────────────── */ + +function formatDuration(ms: number): string { + const seconds = Math.floor(ms / 1000); + const minutes = Math.floor(seconds / 60); + if (minutes > 0) return `${minutes}m ${seconds % 60}s`; + return `${seconds}s`; +} + +function truncate(s: string, max: number): string { + if (s.length <= max) return s; + return s.slice(0, max - 3) + "..."; +} + +/* ── Tool Definition ──────────────────────────────────────────────── */ + +const DeepResearchParams = Type.Object({ + question: Type.String({ + description: "The research question to investigate", + }), + depth: Type.Optional( + Type.Integer({ + description: + "Number of research rounds (1-3). Each round builds on findings from the previous for deeper analysis. Default: 2", + minimum: 1, + maximum: 3, + default: 2, + }), + ), + breadth: Type.Optional( + Type.Integer({ + description: + "Number of search queries per round (1-5). More queries = broader coverage but slower. Default: 3", + minimum: 1, + maximum: 5, + default: 3, + }), + ), + format: Type.Optional( + Type.Union([Type.Literal("markdown"), Type.Literal("structured")], { + description: + 'Output format for the research report. "markdown" for prose with headings, "structured" for detailed hierarchical sections. Default: "markdown"', + default: "markdown", + }), + ), + audience: Type.Optional( + Type.Union( + [ + Type.Literal("general"), + Type.Literal("expert"), + Type.Literal("executive"), + ], + { + description: + "Target audience for the report. 'general' (accessible), 'expert' (technical depth), 'executive' (concise, action-oriented). Default: 'general'", + default: "general", + }, + ), + ), + details: Type.Optional( + Type.Object({ + showRoundDetails: Type.Optional( + Type.Boolean({ + description: + "Include per-round search methodology in the output. Default: false", + }), + ), + }), + ), +}); + +interface ResearchDetails { + rounds: Array<{ + round: number; + queries: string[]; + findingsCount: number; + resultsCount: number; + failedSearches: number; + }>; + totalSearches: number; + totalPagesScraped: number; + durationMs: number; +} + +/* ── Widget Helper ────────────────────────────────────────────────── */ + +/** + * Create a widget state that drives a spinner-based progress widget. + * Returns the state object, the timer, and cleanup function. + */ +function createProgressWidget( + ctx: any, + initialPhase: ResearchPhase = "generating_queries", +) { + const state: { + phase: ResearchPhase; + message: string; + detail: string | undefined; + fraction: number; + round: number | undefined; + totalRounds: number | undefined; + } = { + phase: initialPhase, + message: "Starting...", + detail: undefined, + fraction: 0, + round: undefined, + totalRounds: undefined, + }; + + let widgetTui: { requestRender(): void } | null = null; + let spinnerIdx = 0; + + ctx.ui.setWidget( + "deep-research", + (tui: { requestRender(): void }, _theme: any) => { + widgetTui = tui; + return { + render: (width: number) => { + const spinner = SPINNER_FRAMES[spinnerIdx]; + const icon = PHASE_ICONS[state.phase] ?? ""; + const roundInfo = + state.round && state.totalRounds + ? ` Round ${state.round}/${state.totalRounds}` + : ""; + const firstLine = `${spinner} ${icon} ${state.message}${roundInfo}`; + const lines: string[] = [truncateToWidth(firstLine, width)]; + if (state.detail) { + lines.push(truncateToWidth(` ${state.detail}`, width)); + } + if (state.fraction > 0) { + const barLen = Math.min(15, Math.max(3, width - 4)); + const filled = Math.round(barLen * state.fraction); + const bar = "█".repeat(filled) + "░".repeat(barLen - filled); + lines.push(` ${bar}`); + } + return lines; + }, + invalidate: () => {}, + }; + }, + ); + + const spinnerTimer = setInterval(() => { + spinnerIdx = (spinnerIdx + 1) % SPINNER_FRAMES.length; + widgetTui?.requestRender(); + }, 100); + + const onProgress: ResearchProgress = (update) => { + state.phase = update.phase; + state.message = update.message; + state.detail = update.detail; + state.fraction = update.fraction ?? 0; + state.round = update.round; + state.totalRounds = update.totalRounds; + }; + + const cleanup = () => { + clearInterval(spinnerTimer); + ctx.ui.setWidget("deep-research", undefined); + }; + + return { state, onProgress, cleanup, spinnerTimer }; +} + +/* ── Extension Entry ───────────────────────────────────────────────── */ + +export default function (pi: ExtensionAPI) { + pi.registerTool({ + name: "deep_research", + label: "Deep Research", + description: [ + "Conduct multi-round deep web research on any topic using Firecrawl.", + "Generates diverse search queries, searches the web in parallel, analyzes results,", + "and produces a comprehensive report with numbered citations and a bibliography.", + "Supports iterative refinement and sub-question decomposition for deeper analysis.", + "Parameters: question (required), depth, breadth, format, audience, details.", + "Use for complex, multi-faceted questions that benefit from multiple search angles;", + "for simple fact-finding questions use firecrawl_search directly instead.", + "Set audience to 'executive' for concise, action-oriented reports; 'expert' for technical depth;", + "'general' (default) for accessible reports.", + ].join(" "), + parameters: DeepResearchParams as unknown as TSchema, + + async execute( + _toolCallId: string, + params: { + question: string; + depth?: number; + breadth?: number; + format?: "markdown" | "structured"; + audience?: Audience; + details?: { showRoundDetails?: boolean }; + }, + signal: AbortSignal | undefined, + onUpdate: ((partial: any) => void) | undefined, + ctx: any, + ) { + const config: ResearchConfig = { + question: params.question, + depth: params.depth ?? 2, + breadth: params.breadth ?? 3, + format: params.format ?? "markdown", + audience: params.audience ?? "general", + }; + + const abortSignal = signal; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { state: _state, onProgress, cleanup } = createProgressWidget(ctx); + + let researchResult: ResearchReport | null = null; + let lastError: string | null = null; + + try { + ctx.ui.setStatus( + "deep-research", + `🌐 Researching: ${truncate(config.question, 40)}`, + ); + + onProgress({ + phase: "generating_queries", + message: "Starting deep research...", + fraction: 0, + }); + + researchResult = await runDeepResearch( + config, + ctx, + onProgress, + abortSignal, + ); + + // ── Build the tool result ────────────────────────────────── + + const details: ResearchDetails = { + rounds: researchResult.rounds.map((r) => ({ + round: r.round, + queries: r.queries.map((q) => q.query), + findingsCount: r.findings.length, + resultsCount: r.results.length, + failedSearches: r.failedSearches, + })), + totalSearches: researchResult.totalSearches, + totalPagesScraped: researchResult.totalPagesScraped, + durationMs: researchResult.durationMs, + }; + + // Stream final content via onUpdate before returning + if (onUpdate) { + onUpdate({ + content: [{ type: "text", text: researchResult.finalReport }], + details: { + phase: "complete", + duration: researchResult.durationMs, + rounds: researchResult.rounds.length, + findings: researchResult.rounds.reduce( + (s, r) => s + r.findings.length, + 0, + ), + references: researchResult.references.length, + }, + }); + } + + cleanup(); + ctx.ui.setStatus("deep-research", undefined); + + let output = researchResult.finalReport; + + // Append methodology section if requested + if (params.details?.showRoundDetails) { + output += `\n\n---\n\n## Research Methodology\n\n`; + for (const round of researchResult.rounds) { + output += `### Round ${round.round}\n\n`; + output += `**Queries:**\n`; + for (const q of round.queries) { + output += `- "${q.query}" (${q.angle}) — ${q.rationale}\n`; + } + output += `\n**Results scraped:** ${round.results.length}\n`; + output += `**Findings extracted:** ${round.findings.length}\n`; + if (round.failedSearches > 0) { + output += `**Failed searches:** ${round.failedSearches}\n`; + } + output += `\n`; + } + output += `**Total searches:** ${researchResult.totalSearches}\n`; + output += `**Total pages scraped:** ${researchResult.totalPagesScraped}\n`; + output += `**Sources in bibliography:** ${researchResult.references.length}\n`; + output += `**Duration:** ${formatDuration(researchResult.durationMs)}\n`; + } + + return { + content: [{ type: "text", text: output }], + details, + }; + } catch (error) { + cleanup(); + ctx.ui.setStatus("deep-research", undefined); + + lastError = error instanceof Error ? error.message : String(error); + + return { + content: [ + { + type: "text", + text: `Research failed: ${lastError}`, + }, + ], + details: { + rounds: [], + totalSearches: 0, + totalPagesScraped: 0, + durationMs: 0, + error: lastError, + } as ResearchDetails & { error: string }, + isError: true, + }; + } + }, + + // ── TUI: Render the tool call (collapsed view) ────────────────── + + renderCall( + args: any, + _options: any, + theme: any, + ) { + const question = truncate(args.question ?? "?", 70); + const depth = args.depth ?? 2; + const breadth = args.breadth ?? 3; + const format = args.format ?? "markdown"; + const audience = args.audience ?? "general"; + + const text = + theme.fg("toolTitle", theme.bold("deep_research ")) + + theme.fg("accent", `"${question}"`) + + theme.fg( + "muted", + ` [depth:${depth} breadth:${breadth} ${format} ${audience}]`, + ); + return new Text(text, 0, 0); + }, + + // ── TUI: Render the tool result (expanded/collapsed) ───────────── + + renderResult( + result: any, + { expanded }: { expanded: boolean }, + theme: any, + _context: any, + ) { + const details = result.details as ResearchDetails | undefined; + + if (!details) { + const text = result.content?.[0]?.text ?? "(no output)"; + return new Text(text, 0, 0); + } + + const container = new Box(); + + // ── Collapsed view ──────────────────────────────────────────── + + if (!expanded) { + const totalRounds = details.rounds.length; + const totalFindings = details.rounds.reduce( + (s, r) => s + r.findingsCount, + 0, + ); + const duration = formatDuration(details.durationMs); + + let text = ""; + text += + theme.fg("success", "✓ ") + + theme.fg("toolTitle", theme.bold("deep research")); + text += theme.fg( + "muted", + ` — ${totalRounds} rounds, ${totalFindings} findings`, + ); + text += theme.fg("dim", ` (${duration})`); + text += "\n"; + + for (const round of details.rounds) { + const icon = + round.findingsCount > 0 + ? theme.fg("success", "✓") + : theme.fg("muted", "·"); + text += ` ${icon} ${theme.fg("accent", `Round ${round.round}:`)} `; + text += theme.fg( + "dim", + `${round.queries.length} queries, ${round.resultsCount} pages, ${round.findingsCount} findings`, + ); + text += "\n"; + } + + text += theme.fg("muted", "(Ctrl+O to expand)"); + container.addChild(new Text(text, 0, 0)); + return container; + } + + // ── Expanded view ───────────────────────────────────────────── + + const headerText = + theme.fg("toolTitle", theme.bold("Deep Research Results")) + + "\n" + + theme.fg("dim", `Duration: ${formatDuration(details.durationMs)} | `) + + theme.fg("dim", `Searches: ${details.totalSearches} | `) + + theme.fg("dim", `Pages scraped: ${details.totalPagesScraped}`); + container.addChild(new Text(headerText, 0, 0)); + + for (const round of details.rounds) { + container.addChild(new Text("", 0, 0)); // Spacer + const roundHeader = `Round ${round.round}`; + container.addChild( + new Text(theme.fg("toolTitle", theme.bold(roundHeader)), 0, 0), + ); + container.addChild( + new Text( + theme.fg( + "dim", + `${round.queries.length} queries → ${round.resultsCount} pages → ${round.findingsCount} findings`, + ), + 0, + 0, + ), + ); + for (const q of round.queries) { + container.addChild( + new Text( + theme.fg("muted", " · ") + theme.fg("accent", truncate(q, 70)), + 0, + 0, + ), + ); + } + } + + return container; + }, + }); + + // ── Command ─────────────────────────────────────────────────────── + + pi.registerCommand("deepi-research", { + description: + "Conduct multi-round deep web research on any topic via Firecrawl. Usage: /deepi-research ", + handler: async (args: string, ctx: ExtensionCommandContext) => { + if (!args || args.trim().length === 0) { + ctx.ui.notify( + "Usage: /deepi-research ", + "error", + ); + return; + } + + // Ask about depth + const depthStr = await ctx.ui.select("Research depth?", [ + "1 round (quick survey)", + "2 rounds (standard)", + "3 rounds (deep dive)", + ]); + const depth = depthStr?.startsWith("1") + ? 1 + : depthStr?.startsWith("3") + ? 3 + : 2; + + // Ask about breadth + const breadthStr = await ctx.ui.select("Research breadth?", [ + "1 query/round (narrow)", + "3 queries/round (balanced)", + "5 queries/round (broad)", + ]); + const breadth = breadthStr?.startsWith("1") + ? 1 + : breadthStr?.startsWith("5") + ? 5 + : 3; + + // Ask about audience + const audienceStr = await ctx.ui.select("Report audience?", [ + "General (accessible, explains terms)", + "Expert (technical depth, assumes domain knowledge)", + "Executive (concise, action-oriented)", + ]); + const audience: Audience = audienceStr?.startsWith("Expert") + ? "expert" + : audienceStr?.startsWith("Executive") + ? "executive" + : "general"; + + ctx.ui.setStatus( + "deep-research", + `🌐 Researching: ${truncate(args, 40)}`, + ); + + const config: ResearchConfig = { + question: args, + depth, + breadth, + format: "markdown", + audience, + }; + + const { onProgress, cleanup } = createProgressWidget(ctx); + + try { + const report = await runDeepResearch(config, ctx, onProgress); + + cleanup(); + ctx.ui.setStatus("deep-research", undefined); + + // Show notification + ctx.ui.notify( + `Research complete: ${report.rounds.length} rounds, ${report.totalSearches} searches, ${report.totalPagesScraped} pages, ${report.references.length} sources in ${formatDuration(report.durationMs)}`, + "info", + ); + + // Send the report as a user message + pi.sendUserMessage( + `## Deep Research: ${args}\n\n${report.finalReport}\n\n---\n*${report.rounds.length} rounds · ${report.totalSearches} searches · ${report.totalPagesScraped} pages · ${report.references.length} sources · ${formatDuration(report.durationMs)}*`, + ); + } catch (error) { + cleanup(); + ctx.ui.setStatus("deep-research", undefined); + const msg = error instanceof Error ? error.message : String(error); + ctx.ui.notify(`Research failed: ${msg}`, "error"); + } + }, + }); + + // ── Startup check ───────────────────────────────────────────────── + + pi.on("session_start", async (_event: unknown, ctx: ExtensionContext) => { + const reachable = await isFirecrawlReachable(); + if (!reachable) { + ctx.ui.notify( + "Deep Research: Firecrawl endpoint unreachable — searches will fail. Set firecrawl.baseUrl in config.yml (global ~/.omp/agent or project .omp) or the FIRECRAWL_BASE_URL env var.", + "warning", + ); + } + }); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..9d3afed --- /dev/null +++ b/package.json @@ -0,0 +1,43 @@ +{ + "name": "@mikefreno/omp-deepi-research", + "version": "0.2.0", + "description": "Deep research extension for pi — parallel web research via Firecrawl with iterative query refinement", + "keywords": [ + "pi-package", + "pi-extension", + "research", + "firecrawl", + "deep-research", + "web-search", + "ai" + ], + "license": "MIT", + "files": [ + "index.ts", + "src/", + "README.md", + "LICENSE" + ], + "scripts": { + "typecheck": "tsc --noEmit", + "prepublishOnly": "tsc --noEmit" + }, + "engines": { + "bun": ">=1.3.14" + }, + "dependencies": { + "yaml": "^2.4.0" + }, + "omp": { + "extensions": [ + "./index.ts" + ] + }, + "devDependencies": { + "@oh-my-pi/pi-coding-agent": "17.2.12", + "@oh-my-pi/pi-tui": "17.2.12", + "@types/node": "^20.0.0", + "typebox": "^1.1.0", + "typescript": "^5.3.0" + } +} diff --git a/scripts/run-harness.ts b/scripts/run-harness.ts new file mode 100644 index 0000000..4c39b2f --- /dev/null +++ b/scripts/run-harness.ts @@ -0,0 +1,115 @@ +/** + * Deep Research — standalone end-to-end test harness + * + * Imports the extension's REAL source (fresh, uncached) and runs the full + * research pipeline with real Firecrawl + real pi agent sessions. + * + * WHY: pi caches extension modules per session (keyed by path + cwd + + * generation) and only invalidates on /reload or cwd change. Tool calls in + * a live session therefore run the version loaded at session start. This + * harness bypasses that cache so you can iterate on src/ without reloading + * pi, and it prints verification stats (round stats, angle provenance, + * corroboration distribution, citation integrity, authority stats). + * + * Usage (from repo root): + * NODE_PATH=/opt/homebrew/lib/node_modules bun scripts/run-harness.ts \ + * "" [depth] [breadth] [audience] + * + * Requires the pi SDK to be resolvable (NODE_PATH above points at the + * global pi install) and Firecrawl reachable (settings.json firecrawl.baseUrl). + * Report is written to /tmp/deepi-harness/report.md. + */ +import { runDeepResearch } from "../src/research.ts"; +import type { ResearchReport } from "../src/types.ts"; +import { mkdirSync, writeFileSync } from "node:fs"; + +const question = + process.argv[2] ?? "Compare Rust and Go for backend services in 2025"; +const depth = Number(process.argv[3] ?? 2); +const breadth = Number(process.argv[4] ?? 3); +const audience = (process.argv[5] ?? "expert") as + | "expert" + | "general" + | "executive"; + +const started = Date.now(); + +const report: ResearchReport = await runDeepResearch( + { + question, + depth, + breadth, + format: "markdown", + audience, + }, + { cwd: process.cwd() } as any, + (update) => { + const round = update.round + ? ` [r${update.round}/${update.totalRounds}]` + : ""; + console.log(` [${update.phase}${round}] ${update.message}`); + }, +); + +console.log("\n" + "=".repeat(80)); +console.log(`DURATION: ${((Date.now() - started) / 1000).toFixed(1)}s`); +console.log(`TOTAL SEARCHES: ${report.totalSearches}`); +console.log(`TOTAL PAGES: ${report.totalPagesScraped}`); +console.log(`REFERENCES: ${report.references.length}`); +console.log(`ROUNDS: ${report.rounds.length}`); + +for (const round of report.rounds) { + console.log( + ` Round ${round.round}: ${round.queries.length} queries (${round.successfulSearches} ok, ${round.failedSearches} failed) → ${round.results.length} unique pages → ${round.findings.length} findings`, + ); + const angles = new Map(); + for (const f of round.findings) { + const a = f.angle ?? "none"; + angles.set(a, (angles.get(a) ?? 0) + 1); + } + console.log( + ` finding angles: ${Array.from(angles.entries()) + .map(([a, n]) => `${a}(${n})`) + .join(", ")}`, + ); + const corr = round.findings.map((f) => f.corroborationScore ?? 0); + const strong = corr.filter((c) => c >= 0.5).length; + const partial = corr.filter((c) => c > 0 && c < 0.5).length; + const none = corr.filter((c) => c === 0).length; + console.log( + ` corroboration: ${strong} strong(>=0.5), ${partial} partial, ${none} none`, + ); +} + +// Citation integrity: citations used in the report BODY vs reference list. +// (Reference titles may legitimately contain "[2026]" etc. — those live in +// the ## References section which was rebuilt authoritatively.) +const refIds = new Set(report.references.map((r) => r.id)); +const body = report.finalReport.replace(/^## References[\s\S]*$/m, ""); +const cited = new Set( + [...body.matchAll(/\[(\d+)\]/g)].map((m) => Number(m[1])), +); +const dangling = [...cited].filter((id) => !refIds.has(id)); +console.log( + `CITATIONS (body only): ${cited.size} unique numbers used, ${dangling.length} dangling (${dangling.join(",")})`, +); +console.log( + `REFERENCES SECTION PRESENT: ${/^## References/m.test(report.finalReport)}`, +); +const iconCount = (report.finalReport.match(/[⭐✓○]/g) ?? []).length; +console.log(`AUTHORITY ICONS IN REFERENCES: ${iconCount}`); + +const authorities = report.references.map((r) => r.authorityScore); +const avgAuth = + authorities.reduce((a, b) => a + b, 0) / Math.max(1, authorities.length); +console.log( + `AVG SOURCE AUTHORITY: ${(avgAuth * 100).toFixed(0)}% (max ${(Math.max(...authorities) * 100).toFixed(0)}%, min ${(Math.min(...authorities) * 100).toFixed(0)}%)`, +); +console.log( + "DOMAINS:", + [...new Set(report.references.map((r) => r.domain))].join(", "), +); + +mkdirSync("/tmp/deepi-harness", { recursive: true }); +writeFileSync("/tmp/deepi-harness/report.md", report.finalReport); +console.log("\nReport saved to /tmp/deepi-harness/report.md"); diff --git a/src/agent.ts b/src/agent.ts new file mode 100644 index 0000000..54c966c --- /dev/null +++ b/src/agent.ts @@ -0,0 +1,150 @@ +/** + * Deep Research — Agent Session helper + * + * Uses omp's in-process `createAgentSession` for LLM subtasks + * (query generation, result analysis, report synthesis). + * Pattern borrowed from ralpi's runAgentSession(). + */ +import { + createAgentSession, + AgentRegistry, + SessionManager, +} from "@oh-my-pi/pi-coding-agent"; +import type { AgentSessionEvent } from "@oh-my-pi/pi-coding-agent"; + +/** Aggregate tool usage stats */ +export interface ToolUsage { + read: number; + write: number; + edit: number; + bash: number; + other: number; +} + +export interface AgentResult { + success: boolean; + text: string; + error?: string; + toolUsage: ToolUsage; +} + +/** + * Run a prompt through an in-process omp agent session. + * Non-blocking — the event loop stays responsive. + */ +export async function runAnalysisAgent( + systemPrompt: string, + taskPrompt: string, + cwd: string, + timeoutMs: number = 120_000, + onEvent?: (event: AgentSessionEvent) => void, + signal?: AbortSignal, +): Promise { + const toolUsage: ToolUsage = { + read: 0, + write: 0, + edit: 0, + bash: 0, + other: 0, + }; + + let timeoutHandle: ReturnType | null = null; + if (timeoutMs > 0) { + timeoutHandle = setTimeout(() => { + sessionRef.session?.agent.abort(); + }, timeoutMs); + } + + const sessionRef: { + session?: Awaited>["session"]; + } = {}; + + try { + const result = await createAgentSession({ + cwd, + sessionManager: SessionManager.inMemory(cwd), + toolNames: ["read", "grep", "glob"], + restrictToolNames: true, + disableExtensionDiscovery: true, + skills: [], + promptTemplates: [], + rules: [], + contextFiles: [], + enableMCP: false, + enableLsp: false, + agentRegistry: new AgentRegistry(), + }); + sessionRef.session = result.session; + + const abortHandler = () => result.session.agent.abort(); + signal?.addEventListener("abort", abortHandler, { once: true }); + + let finalText = ""; + let errorMessage: string | undefined; + + const unsubscribe = result.session.subscribe((event: AgentSessionEvent) => { + onEvent?.(event); + + if (event.type === "message_end") { + const message = event.message as { + role?: string; + content?: unknown; + errorMessage?: string; + }; + if (message.role !== "assistant") return; + if (message.errorMessage) errorMessage = message.errorMessage; + const text = extractAssistantText(message.content); + if (text) finalText = text; + } + + if (event.type === "tool_execution_start") { + const name = event.toolName; + if (name in toolUsage) { + (toolUsage as unknown as Record)[name]++; + } else { + toolUsage.other++; + } + } + }); + + if (signal?.aborted) throw new Error("Aborted"); + + await result.session.prompt(`${systemPrompt}\n\n${taskPrompt}`); + await result.session.agent.waitForIdle(); + + unsubscribe(); + result.session.dispose(); + signal?.removeEventListener("abort", abortHandler); + if (timeoutHandle) clearTimeout(timeoutHandle); + + if (errorMessage && !finalText) { + return { success: false, text: "", error: errorMessage, toolUsage }; + } + + return { success: true, text: finalText.trim(), toolUsage }; + } catch (error) { + if (timeoutHandle) clearTimeout(timeoutHandle); + return { + success: false, + text: "", + error: error instanceof Error ? error.message : String(error), + toolUsage, + }; + } finally { + sessionRef.session?.dispose(); + } +} + +function extractAssistantText(content: unknown): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .filter( + (c): c is { type: string; text?: string } => + !!c && + typeof c === "object" && + (c as { type?: string }).type === "text", + ) + .map((c) => (c as { text?: string }).text ?? "") + .join(""); +} diff --git a/src/firecrawl.ts b/src/firecrawl.ts new file mode 100644 index 0000000..03ff965 --- /dev/null +++ b/src/firecrawl.ts @@ -0,0 +1,525 @@ +/** + * Deep Research — direct Firecrawl HTTP client + * + * Calls the self-hosted Firecrawl API directly (same approach as the + * firecrawl.ts extension) + */ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { parse as parseYaml } from "yaml"; +import type { SearchResult, EnrichedSearchResult, ContentType } from "./types"; +import { getAgentDir } from "@oh-my-pi/pi-coding-agent"; + +/* ── Config ──────────────────────────────────────────────────────── */ + +/** + * Read and merge Firecrawl settings from omp's config.yml files. + * + * Resolution order (later wins): + * 1. env vars FIRECRAWL_BASE_URL / FIRECRAWL_API_KEY + * 2. global ~/.omp/agent/config.yml → firecrawl.* + * 3. project .omp/config.yml → firecrawl.* + * 4. default http://localhost:3002 (if no baseUrl configured) + */ +function loadFirecrawlConfig() { + // Start with env var defaults + let baseUrl = process.env.FIRECRAWL_BASE_URL ?? "http://localhost:3002"; + let apiKey = process.env.FIRECRAWL_API_KEY; + + const agentDir = getAgentDir(); + + // Helper: read a config.yml and merge its firecrawl.* keys + const tryReadConfig = (configPath: string): void => { + try { + const raw = parseYaml(fs.readFileSync(configPath, "utf-8")) as Record< + string, + unknown + >; + const fc = (raw?.firecrawl ?? {}) as Record; + if (typeof fc.baseUrl === "string" && fc.baseUrl.length > 0) { + baseUrl = fc.baseUrl; + } + if (typeof fc.apiKey === "string" && fc.apiKey.length > 0) { + apiKey = fc.apiKey; + } + } catch { + // File missing or unparseable — skip + } + }; + + // 1. Global config + tryReadConfig(path.join(agentDir, "config.yml")); + + // 2. Project config (override global) + tryReadConfig(path.join(process.cwd(), ".omp", "config.yml")); + + return { + baseUrl: baseUrl.replace(/\/+$/, ""), + apiKey, + }; +} + +const { baseUrl: BASE_URL, apiKey: API_KEY } = loadFirecrawlConfig(); + +/* ── Domain Authority Heuristics ─────────────────────────────────── */ + +/** + * Known high-authority domains and their authority scores (0.0 – 1.0). + * Academic, official, and established technical sources score highest. + */ +const AUTHORITY_DOMAINS: Record = { + // Academic & scholarly + "arxiv.org": 0.95, + "scholar.google.com": 0.95, + "pubmed.ncbi.nlm.nih.gov": 0.95, + "semanticscholar.org": 0.9, + "ieee.org": 0.95, + "acm.org": 0.95, + "springer.com": 0.9, + "sciencedirect.com": 0.9, + "wiley.com": 0.85, + "nature.com": 0.95, + "science.org": 0.95, + "plos.org": 0.85, + // Official documentation + "docs.python.org": 0.9, + "developer.mozilla.org": 0.9, + "learn.microsoft.com": 0.85, + "developer.apple.com": 0.85, + "kubernetes.io": 0.85, + "react.dev": 0.85, + "nextjs.org": 0.8, + // Official language/platform docs + "go.dev": 0.9, + "golang.org": 0.9, + "rust-lang.org": 0.9, + "nodejs.org": 0.85, + "python.org": 0.85, + "typescriptlang.org": 0.85, + "openai.com": 0.8, + "anthropic.com": 0.8, + "cloud.google.com": 0.8, + "aws.amazon.com": 0.8, + "azure.microsoft.com": 0.8, + "postgresql.org": 0.85, + "sqlite.org": 0.85, + "redis.io": 0.85, + "docker.com": 0.75, + "elastic.co": 0.75, + "grafana.com": 0.75, + "datadoghq.com": 0.75, + "cloudflare.com": 0.8, + "blog.cloudflare.com": 0.8, + "techempower.com": 0.8, + "goframe.org": 0.75, + "corrode.dev": 0.6, + "evrone.com": 0.4, + "rustify.rs": 0.4, + "core.cz": 0.4, + // Medical / clinical + "mayoclinic.org": 0.9, + "heart.org": 0.85, + "researchgate.net": 0.6, + "healthline.com": 0.5, + "medicalnewstoday.com": 0.5, + "webmd.com": 0.45, + "verywellhealth.com": 0.5, + // Databases & dev tools + "mysql.com": 0.85, + "mariadb.org": 0.85, + "cockroachlabs.com": 0.7, + "timescale.com": 0.7, + "mongodb.com": 0.8, + "liquibase.com": 0.6, + "sqlpipe.com": 0.5, + "data-tune.com": 0.4, + "binaryigor.com": 0.4, + // Government & non-profits + ".gov": 0.9, + ".edu": 0.85, + "who.int": 0.9, + "worldbank.org": 0.85, + "oecd.org": 0.85, + // Established tech & news + "github.com": 0.8, + "stackoverflow.com": 0.7, + "medium.com": 0.4, + "dev.to": 0.5, + "wikipedia.org": 0.7, + "reuters.com": 0.8, + "apnews.com": 0.8, + "bbc.com": 0.75, + "nytimes.com": 0.75, + "theguardian.com": 0.7, + "techcrunch.com": 0.6, + "arstechnica.com": 0.65, + "wired.com": 0.65, + "infoworld.com": 0.55, + // Practitioner/aggregator content with measurable quality + "github.io": 0.6, + "crates.io": 0.7, + "docs.rs": 0.75, + "digitalocean.com": 0.6, + "freecodecamp.org": 0.6, + "geeksforgeeks.org": 0.35, + "stackexchange.com": 0.65, + "huggingface.co": 0.65, + "nasa.gov": 0.9, + "mit.edu": 0.9, + "stanford.edu": 0.9, + "harvard.edu": 0.9, + "ox.ac.uk": 0.9, + "cam.ac.uk": 0.9, + // Low-authority: personal social / SEO content + "linkedin.com": 0.25, + "reddit.com": 0.25, + "x.com": 0.3, + "twitter.com": 0.3, + "youtube.com": 0.3, + "blogspot.com": 0.25, + "substack.com": 0.3, + "hashnode.dev": 0.35, + "quora.com": 0.3, + "netguru.com": 0.3, + "relisoftware.com": 0.3, + "dasroot.net": 0.3, + "devgenius.io": 0.3, + "devnewsletter.com": 0.3, +}; + +/** + * Known low-quality SEO/comparison-spam domains. Content is often + * auto-generated, republished from other sites, or thin on substance. + * These get a hard authority floor so they never rank above real content. + */ +const LOW_AUTHORITY_DOMAINS: Record = { + "markaicode.com": 0.15, + "bytegoblin.io": 0.2, + "towardsdev.com": 0.2, + "rustvsgo.com": 0.3, + "seekingalpha.com": 0.3, + "investopedia.com": 0.55, + "devops-daily.com": 0.3, +}; + +/** Content-type hints based on domain patterns */ +const CONTENT_TYPE_HINTS: [RegExp, ContentType][] = [ + [ + /arxiv\.org|semanticscholar|ieee\.org|acm\.org|springer|sciencedirect|pubmed\.ncbi/, + "paper", + ], + [ + /docs\.|learn\.|developer\.|kubernetes\.io|react\.dev|nextjs\.org/, + "documentation", + ], + [/wikipedia\.org|stackoverflow\.com|medium\.com|dev\.to/, "forum"], + [ + /reuters\.com|apnews\.com|bbc\.com|nytimes\.com|techcrunch|arstechnica|wired/, + "news", + ], + [/\.gov|\.edu|who\.int|worldbank|oecd\.org/, "official"], + [/github\.com/, "documentation"], +]; + +/* ── Source enrichment helpers ───────────────────────────────────── */ + +/** + * Extract the registered domain from a URL (e.g., "blog.example.com" → "example.com"). + * Uses a simple 2-part TLD heuristic. For common cases like .co.uk this is approximate. + */ +function extractDomain(url: string): string { + try { + const hostname = new URL(url).hostname.toLowerCase(); + // Special-case common multi-part TLDs + const multiPartTlds = + /\.(co\.uk|org\.uk|ac\.uk|gov\.uk|com\.au|co\.jp|co\.kr|com\.br)$/; + const parts = hostname.split("."); + if (multiPartTlds.test(hostname) && parts.length >= 3) { + return parts.slice(-3).join("."); + } + return parts.slice(-2).join("."); + } catch { + return url.replace(/^https?:\/\//, "").split("/")[0] ?? url; + } +} + +function computeAuthorityScore(domain: string): number { + // Hard floor for known low-authority domains first + if (LOW_AUTHORITY_DOMAINS[domain] !== undefined) + return LOW_AUTHORITY_DOMAINS[domain]; + + // Direct match first + if (AUTHORITY_DOMAINS[domain]) return AUTHORITY_DOMAINS[domain]; + + // Suffix matches (.gov, .edu, etc.) + for (const [key, score] of Object.entries(AUTHORITY_DOMAINS)) { + if (key.startsWith(".") && domain.endsWith(key)) return score; + } + + // Subdomain matches (e.g., blog.example.com matches example.com) + const parent = domain.split(".").slice(-2).join("."); + if (parent !== domain && AUTHORITY_DOMAINS[parent]) { + return AUTHORITY_DOMAINS[parent] * 0.9; + } + + // github.io personal sites: treat as practitioner content (medium) + if (domain.endsWith(".github.io")) return 0.55; + + return 0.3; // Unknown / low-authority default +} + +function detectContentType(url: string, description: string): ContentType { + const lowerUrl = url.toLowerCase(); + const lowerDesc = description.toLowerCase(); + + for (const [pattern, type] of CONTENT_TYPE_HINTS) { + if (pattern.test(lowerUrl)) return type; + } + + // Heuristics from description text + if (/paper|research|study|experiment|analysis\b/.test(lowerDesc)) + return "paper"; + if (/documentation|guide|tutorial|api|reference/.test(lowerDesc)) + return "documentation"; + if (/blog|post|article|opinion/.test(lowerDesc)) return "blog"; + if (/news|report|announce|release/.test(lowerDesc)) return "news"; + if (/forum|discussion|question|answer|thread/.test(lowerDesc)) return "forum"; + + return "other"; +} + +function tryParseDate(dateStr: string | undefined | null): Date | null { + if (!dateStr) return null; + const d = new Date(dateStr); + return isNaN(d.getTime()) ? null : d; +} + +/** + * Normalize a title for near-duplicate detection: lowercase, strip + * punctuation, collapse whitespace, drop common filler words. + * Two syndicated copies of the same article normalize identically. + */ +export function normalizeTitle(title: string): string { + return title + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .replace( + /\b(?:the|a|an|of|for|and|or|in|on|with|vs|versus|to|how|what|why|2024|2025|2026)\b/g, + " ", + ) + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Near-duplicate check between two titles: normalized forms must share + * a substantial token overlap (same core words in the same order). + */ +export function isNearDuplicateTitle(a: string, b: string): boolean { + const normA = normalizeTitle(a); + const normB = normalizeTitle(b); + if (!normA || !normB) return false; + if (normA === normB) return true; + + const tokensA = normA.split(" "); + const tokensB = normB.split(" "); + if (tokensA.length < 3 || tokensB.length < 3) return normA === normB; + + // Check if one title is a substring of the other (after normalization) + if (normA.includes(normB) || normB.includes(normA)) return true; + + // Jaccard-ish overlap on the shorter token set + const [short, long] = + tokensA.length <= tokensB.length ? [tokensA, tokensB] : [tokensB, tokensA]; + const overlap = short.filter((t) => long.includes(t)).length; + return overlap / short.length >= 0.75; +} + +/** + * Enrich a raw search result with source authority metadata. + * Accepts extra fields (e.g. date) from the Firecrawl API response. + */ +export function enrichResult( + result: SearchResult & Record, +): EnrichedSearchResult { + const domain = extractDomain(result.url); + return { + ...result, + domain, + authorityScore: computeAuthorityScore(domain), + publishedDate: tryParseDate(result.date as string | undefined), + contentType: detectContentType(result.url, result.description), + }; +} + +/* ── Helpers ──────────────────────────────────────────────────────── */ + +async function firecrawlRequest( + endpoint: string, + body: Record, + signal?: AbortSignal, +): Promise { + const headers: Record = { + "Content-Type": "application/json", + }; + if (API_KEY) { + headers["Authorization"] = `Bearer ${API_KEY}`; + } + + const res = await fetch(`${BASE_URL}/v1/${endpoint}`, { + method: "POST", + headers, + body: JSON.stringify(body), + signal, + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error( + `Firecrawl ${endpoint} failed (${res.status}): ${text.slice(0, 500)}`, + ); + } + + return res.json(); +} + +/** + * firecrawlRequest with retry-with-backoff for transient failures + * (429 rate limits, 5xx server errors, network blips). Does NOT retry + * 4xx client errors (invalid requests) or aborts. + */ +async function firecrawlRequestWithRetry( + endpoint: string, + body: Record, + signal?: AbortSignal, + retries: number = 2, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt <= retries; attempt++) { + if (signal?.aborted) throw new Error("Aborted"); + try { + return await firecrawlRequest(endpoint, body, signal); + } catch (error) { + lastError = error; + const status = + error instanceof Error + ? Number(/failed \((\d+)\)/.exec(error.message)?.[1] ?? 0) + : 0; + // Don't retry aborts or 4xx client errors (other than 429) + if ( + signal?.aborted || + (status >= 400 && status < 500 && status !== 429) + ) { + throw error; + } + if (attempt < retries) { + const delayMs = 400 * 2 ** attempt + Math.random() * 200; + await new Promise((r) => setTimeout(r, delayMs)); + } + } + } + throw lastError; +} + +export async function isFirecrawlReachable(): Promise { + try { + const res = await fetch(`${BASE_URL}/v1/scrape`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(API_KEY ? { Authorization: `Bearer ${API_KEY}` } : {}), + }, + body: JSON.stringify({ url: "https://example.com", formats: ["links"] }), + signal: AbortSignal.timeout(10_000), + }); + return res.ok; + } catch { + return false; + } +} + +/* ── Search ───────────────────────────────────────────────────────── */ + +/** + * Search the web and return structured, enriched results. + * Uses Firecrawl's search endpoint with scrape to get full page content. + */ +export async function searchWeb( + query: string, + limit: number = 5, + signal?: AbortSignal, +): Promise { + const body: Record = { + query, + limit: Math.min(limit, 10), + scrapeOptions: { + formats: ["markdown"], + onlyMainContent: true, + }, + }; + + const result = await firecrawlRequestWithRetry("search", body, signal); + + if (!result || typeof result !== "object") return []; + + const res = result as { + success?: boolean; + data?: Record[]; + error?: string; + }; + + if (!res.success || !res.data) return []; + + const rawResults: (SearchResult & Record)[] = res.data + .map((doc) => ({ + title: (doc.title as string) ?? "", + url: (doc.url as string) ?? "", + description: (doc.description as string) ?? "", + markdown: (doc.markdown as string) ?? "", + // Preserve extra fields for date extraction + ...doc, + })) + .filter((r) => { + // Keep results with a meaningful body OR a substantive description. + // Filters out stub pages / pure navigation results that would + // waste analysis tokens. + const hasBody = (r.markdown ?? "").trim().length >= 150; + const hasSubstantiveDesc = (r.description ?? "").trim().length >= 40; + return hasBody || hasSubstantiveDesc; + }); + + // Enrich each result with source metadata + return rawResults.map(enrichResult); +} + +/* ── Scrape ───────────────────────────────────────────────────────── */ + +/** + * Scrape a single URL and return its markdown content. + */ +export async function scrapeUrl( + url: string, + signal?: AbortSignal, +): Promise<{ title: string; markdown: string; links: string[] } | null> { + const result = await firecrawlRequestWithRetry( + "scrape", + { url, formats: ["markdown"] }, + signal, + ); + + if (!result || typeof result !== "object") return null; + + const res = result as { + success?: boolean; + data?: Record; + error?: string; + }; + + if (!res.success || !res.data) return null; + + return { + title: (res.data.title as string) ?? "", + markdown: (res.data.markdown as string) ?? "", + links: (res.data.links as string[]) ?? [], + }; +} diff --git a/src/queries.ts b/src/queries.ts new file mode 100644 index 0000000..72b639e --- /dev/null +++ b/src/queries.ts @@ -0,0 +1,625 @@ +/** + * Deep Research — Search query generation & refinement + * + * Uses an LLM agent to generate search queries from different research + * angles, then analyzes results to produce follow-up queries. + */ +import type { + SearchQuery, + Finding, + ResearchRound, + EnrichedSearchResult, +} from "./types"; +import { runAnalysisAgent } from "./agent"; + +/* ── System Prompts ──────────────────────────────────────────────── */ + +const DECOMPOSE_SYSTEM = `You are a research methodology expert. Given a broad research question, your job is to break it down into 4-7 focused sub-questions that, when answered, collectively provide a complete answer to the original question. + +Guidelines: +- Each sub-question should tackle ONE specific facet of the research question +- Cover different dimensions: what, how, why, who, comparison, evidence, implications +- Sub-questions should be independently researchable via web search +- Avoid overlap between sub-questions +- Prioritize questions that will surface concrete evidence over speculative ones + +Output ONLY a JSON array of sub-question strings. + +Example: +Input: "What are the benefits and risks of artificial intelligence in healthcare?" +Output: ["What specific AI technologies are currently deployed in clinical healthcare settings?", "What peer-reviewed evidence exists for AI improving diagnostic accuracy?", "What are the documented risks and failure cases of AI in healthcare?", "How do regulatory frameworks (FDA, EMA) address AI-based medical devices?", "What do healthcare practitioners report as barriers to AI adoption?"] +`; + +const GENERATE_QUERIES_SYSTEM = `You are a research methodology expert. Your role is to generate effective web search queries that will yield high-quality, diverse information about a research topic. + +Guidelines: +- Create queries from DIFFERENT angles (technical, practical, comparative, critical, forward-looking, authoritative) +- Each query should target a specific facet of the question +- Queries should use keywords that search engines rank well (avoid overly long questions) +- Cover contrasting viewpoints and alternative approaches +- Include queries for finding authoritative sources (docs, papers, official sites) +- Prioritize recent information where relevant + +Output ONLY a JSON array of objects with fields: +- "query": the search query string +- "rationale": why this query will help answer the research question +- "angle": one of "technical" | "practical" | "comparative" | "critical" | "forward-looking" | "authoritative" | "historical" | "case-study" | "data-statistics" | "ethical" + +Example: +[ + {"query": "Rust async/await performance benchmarks 2024", "rationale": "Understanding current performance characteristics", "angle": "technical"}, + {"query": "Rust vs Go concurrency patterns comparison", "rationale": "Comparative analysis helps contextualize trade-offs", "angle": "comparative"} +] +`; + +const FOLLOWUP_SYSTEM = `You are a research analyst. Given the research question, sub-questions, and findings so far, your job is to identify what's still unknown and generate follow-up search queries to fill those gaps. + +Look for: +- Claims made without sufficient evidence +- Conflicting information that needs resolution +- Angles that haven't been explored yet +- Missing authoritative sources (papers, official docs, primary data) +- Practical implications that need more detail +- Recent developments that might have updated findings + +Guidelines: +- Do NOT repeat or paraphrase queries already explored — aim for genuinely new angles +- Prefer querying for authoritative/primary sources over more blog posts when evidence is weak +- When findings conflict, craft a query designed to resolve the contradiction +- Keep queries concise and keyword-rich + +Output ONLY a JSON array of objects with fields: +- "query": the search query string +- "rationale": what gap this query fills or what angle it explores +- "angle": one of "technical" | "practical" | "comparative" | "critical" | "forward-looking" | "authoritative" | "historical" | "case-study" | "data-statistics" | "ethical" +`; + +/* ── JSON parsing helpers ────────────────────────────────────────── */ + +/** + * Robustly parse a JSON array from LLM output. + * + * LLMs frequently wrap JSON in ```json fences, prepend prose like + * "Here are the queries:", or emit trailing punctuation. This strips + * fences and extracts the first bracketed array before parsing. + * + * Returns null when no array can be extracted. + */ +function parseJsonArray(text: string): unknown[] | null { + if (!text) return null; + + // Strip markdown code fences + const withoutFences = text + .replace(/```(?:json|javascript)?\s*/gi, "") + .replace(/```/g, ""); + + // Find the first '[' ... ']' block (arrays are our target shape) + const start = withoutFences.indexOf("["); + const end = withoutFences.lastIndexOf("]"); + if (start === -1 || end === -1 || end <= start) return null; + + const candidate = withoutFences.slice(start, end + 1); + try { + const parsed = JSON.parse(candidate); + return Array.isArray(parsed) ? parsed : null; + } catch { + // Try to salvage: strip trailing commas (common LLM artifact) + try { + const fixed = candidate.replace(/,\s*([}\]])/g, "$1"); + const parsed = JSON.parse(fixed); + return Array.isArray(parsed) ? parsed : null; + } catch { + return null; + } + } +} + +/** Map a parsed array entry to a SearchQuery, tolerating missing fields. */ +function toSearchQuery(q: Record): SearchQuery | null { + const query = String(q.query ?? "").trim(); + if (!query) return null; + return { + query, + rationale: String(q.rationale ?? "").trim(), + angle: String(q.angle ?? "technical").trim() || "technical", + }; +} + +/* ── Sub-Question Decomposition ───────────────────────────────────── */ + +/** + * Decompose a broad research question into focused, independently + * researchable sub-questions. Returns the sub-questions or an empty + * array if the LLM call fails. + */ +export async function decomposeQuestion( + question: string, + cwd: string, + signal?: AbortSignal, +): Promise { + const taskPrompt = `Break down this research question into 4-7 focused sub-questions:\n\n${question}`; + + const result = await runAnalysisAgent( + DECOMPOSE_SYSTEM, + taskPrompt, + cwd, + 60_000, + undefined, + signal, + ); + + if (!result.success || !result.text) return []; + + const parsed = parseJsonArray(result.text); + if (parsed) { + const subQuestions = parsed + .map(String) + .map((s: string) => s.trim()) + .filter((s: string) => s.length > 10); + if (subQuestions.length > 0) return subQuestions; + } + + return []; +} + +/* ── Query Generation ────────────────────────────────────────────── */ + +/** + * Generate initial search queries for a research question. + * When sub-questions are available, generates queries per sub-question + * for better depth and diversity. + */ +export async function generateQueries( + question: string, + count: number, + cwd: string, + signal?: AbortSignal, + subQuestions?: string[], +): Promise { + // If we have sub-questions, generate queries distributed across them + if (subQuestions && subQuestions.length > 0) { + const queriesPerSub = Math.max(1, Math.ceil(count / subQuestions.length)); + const allQueries: SearchQuery[] = []; + + for (const subQ of subQuestions) { + if (allQueries.length >= count) break; + + const taskPrompt = `Research question: ${question}\nSub-question: ${subQ}\n\nGenerate ${queriesPerSub} search query(ies) to answer this sub-question specifically.`; + + const result = await runAnalysisAgent( + GENERATE_QUERIES_SYSTEM, + taskPrompt, + cwd, + 60_000, + undefined, + signal, + ); + + if (!result.success || !result.text) continue; + + const parsed = parseJsonArray(result.text); + if (parsed) { + const queries = parsed + .slice(0, queriesPerSub) + .map((q) => toSearchQuery(q as Record)) + .filter((q): q is SearchQuery => q !== null); + allQueries.push(...queries); + } + } + + if (allQueries.length > 0) { + return allQueries.slice(0, count); + } + } + + // Fall through to standard query generation + const taskPrompt = `Research question: ${question} + +Generate ${count} diverse search queries to research this topic effectively. Cover different angles.`; + + const result = await runAnalysisAgent( + GENERATE_QUERIES_SYSTEM, + taskPrompt, + cwd, + 60_000, + undefined, + signal, + ); + + if (!result.success || !result.text) { + return generateFallbackQueries(question, count); + } + + try { + const parsed = parseJsonArray(result.text); + if (parsed && parsed.length > 0) { + return parsed + .slice(0, count) + .map((q) => toSearchQuery(q as Record)) + .filter((q): q is SearchQuery => q !== null); + } + } catch { + // JSON parse failed, fall back + } + + return generateFallbackQueries(question, count); +} + +/* ── Follow-up Query Generation ──────────────────────────────────── */ + +/** + * Generate follow-up queries based on findings from previous rounds. + */ +export async function generateFollowUpQueries( + question: string, + rounds: ResearchRound[], + count: number, + cwd: string, + signal?: AbortSignal, +): Promise { + // Build a summary of findings so far + const allFindings = rounds.flatMap((r) => r.findings); + const findingsSummary = allFindings + .map((f) => { + const corr = + f.corroborationScore !== undefined + ? ` [corroboration: ${(f.corroborationScore * 100).toFixed(0)}%]` + : ""; + return `- ${f.title}: ${f.summary} (confidence: ${f.confidence}${corr})`; + }) + .join("\n"); + + const exploredAngles = rounds + .flatMap((r) => r.queries) + .map((q) => `[${q.angle}] ${q.query} — ${q.rationale}`) + .join("\n"); + + // Find low-corroboration or low-confidence topics + const gaps = allFindings + .filter((f) => f.confidence === "low" || (f.corroborationScore ?? 1) < 0.5) + .map((f) => `Gap: ${f.title} — ${f.summary}`) + .join("\n"); + + const taskPrompt = `Research question: ${question} + +Queries already explored: +${exploredAngles} + +Findings so far: +${findingsSummary} + +${gaps ? `Remaining knowledge gaps:\n${gaps}` : ""} + +Generate ${count} follow-up search queries to fill remaining gaps and deepen the research. Do not repeat or paraphrase the queries already explored.`; + + const result = await runAnalysisAgent( + FOLLOWUP_SYSTEM, + taskPrompt, + cwd, + 60_000, + undefined, + signal, + ); + + if (!result.success || !result.text) { + return []; + } + + const exploredNormalized = new Set( + rounds.flatMap((r) => r.queries).map((q) => normalizeQueryText(q.query)), + ); + + const parsed = parseJsonArray(result.text); + if (parsed && parsed.length > 0) { + const fresh: SearchQuery[] = []; + for (const q of parsed.slice(0, count)) { + const sq = toSearchQuery(q as Record); + if (!sq) continue; + const normalized = normalizeQueryText(sq.query); + // Skip queries that are near-duplicates of already-explored ones + if (exploredNormalized.has(normalized)) continue; + if (fresh.some((fq) => normalizeQueryText(fq.query) === normalized)) + continue; + exploredNormalized.add(normalized); + fresh.push(sq); + } + return fresh; + } + + return []; +} + +/** + * Lightweight query-text normalization for duplicate detection. + */ +function normalizeQueryText(query: string): string { + return query + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +/* ── Fallback Query Generation ────────────────────────────────────── */ + +/** + * Fallback query generation when the LLM call fails. + */ +function generateFallbackQueries( + question: string, + count: number, +): SearchQuery[] { + const queries: SearchQuery[] = []; + const angles = [ + { angle: "technical", desc: "technical details and specifications" }, + { + angle: "practical", + desc: "practical examples, tutorials, and best practices", + }, + { angle: "comparative", desc: "comparisons with alternatives" }, + { angle: "critical", desc: "limitations, challenges, and criticisms" }, + { angle: "forward-looking", desc: "future trends and developments" }, + ]; + + for (let i = 0; i < Math.min(count, angles.length); i++) { + queries.push({ + query: `${question} ${angles[i].desc}`, + rationale: `Exploring ${angles[i].desc} related to the research question`, + angle: angles[i].angle as SearchQuery["angle"], + }); + } + + return queries; +} + +/* ── Analysis ────────────────────────────────────────────────────── */ + +const ANALYZE_SYSTEM = `You are a research analyst. Given search results for a specific query, extract key findings. + +For each finding: +- Give it a concise, specific title (a claim, not a topic) +- Summarize what was found in 1-3 sentences, focused on evidence +- List which source URLs support this finding +- Include 1-2 key quotes from the sources +- Rate your confidence (high/medium/low) based on source authority and consistency + +Guidelines: +- Extract 3-6 findings maximum, prioritizing the most decision-relevant +- Prefer findings with concrete evidence over generic observations +- Ignore boilerplate, navigation text, and irrelevant tangents in the content +- Do NOT invent quotes — only use text that appears in the provided content +- When sources conflict, note the conflict in the summary + +Output ONLY a JSON array of objects with fields: +- "title": concise finding title +- "summary": 1-3 sentence summary +- "sources": array of source URLs +- "keyQuotes": array of 1-2 key quotes +- "confidence": "high" | "medium" | "low"`; + +/** + * Analyze search results for a specific query and extract findings. + */ +export async function analyzeResults( + query: string, + results: EnrichedSearchResult[], + cwd: string, + signal?: AbortSignal, + angle?: string, +): Promise { + // Include authority metadata in the prompt so the LLM can consider source quality. + // Token budget: give high-authority sources generous space, truncate + // low-authority/SEO content aggressively so junk doesn't dominate the prompt. + const MAX_CHARS_HIGH_AUTH = 3500; + const MAX_CHARS_LOW_AUTH = 1200; + + const resultsText = results + .map((r, i) => { + const maxChars = + r.authorityScore >= 0.6 ? MAX_CHARS_HIGH_AUTH : MAX_CHARS_LOW_AUTH; + const content = r.markdown.slice(0, maxChars).trim(); + const body = + content.length > 0 + ? content + : `(no body content; description only)\n${r.description}`; + return `--- Result ${i + 1} ---\nTitle: ${r.title}\nURL: ${r.url}\nDomain: ${r.domain}\nAuthority Score: ${(r.authorityScore * 100).toFixed(0)}%\nContent Type: ${r.contentType}\nDescription: ${r.description}\nContent:\n${body}`; + }) + .join("\n\n"); + + const taskPrompt = `Search query: "${query}"${angle ? ` (angle: ${angle})` : ""} + +Search results: +${resultsText} + +Extract key findings from these results. Consider source authority when rating confidence.`; + + const result = await runAnalysisAgent( + ANALYZE_SYSTEM, + taskPrompt, + cwd, + 90_000, + undefined, + signal, + ); + + if (!result.success || !result.text) return []; + + const parsed = parseJsonArray(result.text); + if (parsed) { + return parsed + .map((f) => { + const entry = f as Record; + return { + title: String(entry.title ?? "").trim(), + summary: String(entry.summary ?? "").trim(), + sources: Array.isArray(entry.sources) + ? entry.sources.map(String) + : [], + keyQuotes: Array.isArray(entry.keyQuotes) + ? entry.keyQuotes.map(String) + : [], + confidence: (["high", "medium", "low"].includes( + String(entry.confidence), + ) + ? String(entry.confidence) + : "medium") as Finding["confidence"], + // Provenance: which query and angle produced this finding + query, + angle, + }; + }) + .filter((f) => f.title && f.summary); + } + + return []; +} + +/* ── Corroboration Tracking ──────────────────────────────────────── */ + +/** + * Cross-reference all findings to compute corroboration scores. + * + * For each finding, we check: + * 1. How many other findings reference the same or similar source URLs + * 2. The authority scores of the supporting sources + * 3. Whether independent domains support the same claim + * + * Returns the findings with added corroborationScore, bestSourceAuthority, + * and avgSourceAuthority. + */ +export function computeCorroboration( + findings: Finding[], + urlQueryCounts?: Map, +): Finding[] { + if (findings.length === 0) return []; + + // Collect all unique source URLs and their authority scores + // In a real implementation, we'd map URLs to EnrichedSearchResult authority scores + // For now, extract domain-level patterns + + // Build a map of domain -> authority scores from source URLs + const domainAuthority = new Map(); + for (const finding of findings) { + for (const url of finding.sources) { + try { + const domain = extractDomainSimple(url); + if (!domainAuthority.has(domain)) { + domainAuthority.set(domain, heuristicDomainScore(domain)); + } + } catch { + // skip invalid URLs + } + } + } + + return findings.map((finding) => { + if (finding.sources.length === 0) { + return { + ...finding, + corroborationScore: 0, + bestSourceAuthority: 0, + avgSourceAuthority: 0, + }; + } + + // Compute source authority stats + const authorities: number[] = finding.sources.map((url) => { + try { + const domain = extractDomainSimple(url); + return domainAuthority.get(domain) ?? 0.3; + } catch { + return 0.3; + } + }); + + const bestAuthority = Math.max(...authorities); + const avgAuthority = + authorities.reduce((a, b) => a + b, 0) / authorities.length; + + // Compute corroboration. + // + // PRIMARY signal (when urlQueryCounts is provided): what fraction of + // this finding's sources were independently surfaced by multiple + // DIFFERENT search queries? A source found by several independent + // searches is genuinely corroborated; same-query duplicates do not + // count (findings from one query analyzed the same result set). + // + // FALLBACK signal (no map): domain-level agreement across findings + // from different queries. + let corroborationScore: number; + + if (urlQueryCounts && urlQueryCounts.size > 0) { + const multiQuerySources = finding.sources.filter( + (url) => (urlQueryCounts.get(url) ?? 1) > 1, + ).length; + corroborationScore = + finding.sources.length > 0 + ? multiQuerySources / finding.sources.length + : 0; + } else { + // Fallback: cross-query agreement by shared domain + const myDomains = new Set( + finding.sources.map((u) => extractDomainSimple(u)), + ); + let corroboratingFindings = 0; + let independentOthers = 0; + + for (const other of findings) { + if (other === finding) continue; + // Same query provenance = same analyzed result set = not independent + if (other.query && finding.query && other.query === finding.query) { + continue; + } + independentOthers++; + const otherDomains = new Set( + other.sources.map((u) => extractDomainSimple(u)), + ); + const shared = [...myDomains].some((d) => otherDomains.has(d)); + if (shared) corroboratingFindings++; + } + + corroborationScore = + independentOthers > 0 + ? Math.min(1, corroboratingFindings / independentOthers) + : 0; + } + + return { + ...finding, + corroborationScore: Math.round(corroborationScore * 100) / 100, + bestSourceAuthority: Math.round(bestAuthority * 100) / 100, + avgSourceAuthority: Math.round(avgAuthority * 100) / 100, + }; + }); +} + +/** + * Simple domain extraction (avoids URL constructor for compatibility). + */ +function extractDomainSimple(url: string): string { + const match = url.match(/https?:\/\/([^/]+)/); + if (!match) return url; + const hostname = match[1].toLowerCase(); + const parts = hostname.split("."); + const multiPartTlds = + /\.(co\.uk|org\.uk|ac\.uk|gov\.uk|com\.au|co\.jp|co\.kr|com\.br)$/; + if (multiPartTlds.test(hostname) && parts.length >= 3) { + return parts.slice(-3).join("."); + } + return parts.slice(-2).join("."); +} + +/** + * Very basic domain score heuristic without the full domain list. + */ +function heuristicDomainScore(domain: string): number { + if (/\.gov$|\.edu$/.test(domain)) return 0.85; + if (/arxiv|scholar|pubmed|ieee|acm|springer|nature|science/.test(domain)) + return 0.9; + if (/github|gitlab|bitbucket/.test(domain)) return 0.75; + if (/wikipedia|stackoverflow|medium|dev\.to/.test(domain)) return 0.55; + if (/docs\.|learn\.|developer\./.test(domain)) return 0.8; + if (/reuters|apnews|bbc|nytimes|bloomberg/.test(domain)) return 0.75; + if (/blog|forum|reddit/.test(domain)) return 0.3; + return 0.4; +} diff --git a/src/report.ts b/src/report.ts new file mode 100644 index 0000000..7a15c7c --- /dev/null +++ b/src/report.ts @@ -0,0 +1,530 @@ +/** + * Deep Research — Report synthesis + * + * Takes all research rounds and synthesizes a comprehensive report + * using an LLM agent. Produces: + * - Numbered inline citations with a bibliography + * - Layered report: TL;DR → Executive Summary → Key Findings + * → Detailed Analysis → Limitations/Gaps → References + * - Audience-aware tone adjustment + */ +import type { + ResearchRound, + ResearchConfig, + Reference, + Finding, +} from "./types"; +import { runAnalysisAgent } from "./agent"; +import { isNearDuplicateTitle } from "./firecrawl"; + +/** Return shape from synthesizeReport */ +export interface SynthesisResult { + report: string; + references: Reference[]; +} + +/** + * Maximum findings included in the synthesis prompt. + * Keeps token usage bounded and forces the synthesizer to focus on + * the highest-quality evidence. + */ +const MAX_SYNTHESIS_FINDINGS = 30; + +/** + * Rank a finding for inclusion in synthesis: authority-weighted, + * confidence-weighted, with a corroboration bonus. + */ +function findingQualityScore(f: Finding): number { + const authority = + (f.bestSourceAuthority ?? f.avgSourceAuthority ?? 0.5) || 0.5; + const confidenceWeight = + f.confidence === "high" ? 1.0 : f.confidence === "medium" ? 0.7 : 0.45; + const corroborationBonus = (f.corroborationScore ?? 0) * 0.3; + return authority * confidenceWeight + corroborationBonus; +} + +/* ── System Prompts ──────────────────────────────────────────────── */ + +function buildSynthesisSystem(audience: string): string { + const audienceGuidance: Record = { + expert: + "Assume expert-level domain knowledge. Use precise technical terminology, reference specific methodologies and standards, and prioritize depth over hand-holding. The reader understands the field.", + general: + "Write for an informed general audience. Define technical terms on first use, explain context, and keep the tone accessible but not simplistic. Avoid jargon without explanation.", + executive: + "Write for a busy executive or decision-maker. Lead with actionable conclusions and recommendations. Be concise — use bold for key takeaways. Minimize technical detail; focus on implications, trade-offs, and decisions. Target 2-3 pages.", + }; + + const guidance = audienceGuidance[audience] ?? audienceGuidance.general; + + return `You are a senior research analyst synthesizing findings from multiple web searches into a comprehensive, well-structured report. + +Audience: ${guidance} + +Report structure (use ## headings): +1. **TL;DR** — One paragraph (2-3 sentences) giving the single most important answer +2. **Executive Summary** — 2-3 paragraphs covering what was found, how confident we are, and key implications +3. **Key Findings** — Tiered by importance/confidence. Bullet points with inline citations +4. **Detailed Analysis** — Organized by theme. Each section covers one aspect with evidence +5. **Limitations & Knowledge Gaps** — What evidence is weak, missing, or contradictory +6. **Conclusion** — Wrap up with actionable takeaways + +Citation rules: +- Use numbered references like [1], [2] etc. throughout the text +- At the end, include a ## References section listing each citation +- Format references as: [1] Title — Domain (URL) +- Cite specific evidence, not vague associations +- When multiple sources support a claim, cite all of them: [1][3][5] + +Style guidelines: +- Write in an objective, authoritative tone +- Use bullet points for listing evidence +- Note the confidence level for key claims +- Be thorough but concise — every paragraph should add value +- Use > for notable direct quotes with citations`; +} + +/* ── Evidence Builder ────────────────────────────────────────────── */ + +function buildEvidenceText( + question: string, + rounds: ResearchRound[], +): { evidenceText: string; referenceMap: Map } { + const allFindings = rounds.flatMap((r) => r.findings); + const totalSearches = rounds.reduce((sum, r) => sum + r.queries.length, 0); + const totalPages = rounds.reduce((sum, r) => sum + r.results.length, 0); + + // Build a bibliography map (url -> Reference) + const seenUrls = new Map(); + let refId = 0; + + for (const round of rounds) { + for (const result of round.results) { + if (!seenUrls.has(result.url)) { + refId++; + seenUrls.set(result.url, { + id: refId, + url: result.url, + title: result.title, + domain: result.domain, + authorityScore: result.authorityScore, + accessedAt: new Date().toISOString().split("T")[0], + }); + } + } + } + + // ── Deduplicate findings across rounds ──────────────────────────── + // The same claim often surfaces in multiple rounds under slightly + // different titles. Merge them (union of sources/quotes, keep the + // highest-quality version) so the synthesizer isn't double-counting. + const deduped: Finding[] = []; + for (const finding of allFindings) { + const dupIndex = deduped.findIndex( + (f) => + f.title !== finding.title && + isNearDuplicateTitle(f.title, finding.title), + ); + if (dupIndex === -1) { + deduped.push({ ...finding }); + } else { + const existing = deduped[dupIndex]; + deduped[dupIndex] = { + title: existing.title, + summary: existing.summary, + sources: Array.from(new Set([...existing.sources, ...finding.sources])), + keyQuotes: Array.from( + new Set([...existing.keyQuotes, ...finding.keyQuotes]), + ).slice(0, 3), + confidence: + existing.confidence === "high" || finding.confidence === "high" + ? "high" + : existing.confidence === "medium" || + finding.confidence === "medium" + ? "medium" + : "low", + query: existing.query ?? finding.query, + angle: existing.angle ?? finding.angle, + }; + } + } + + // ── Rank and cap findings for synthesis ─────────────────────────── + const ranked = deduped + .map((f) => ({ f, score: findingQualityScore(f) })) + .sort((a, b) => b.score - a.score) + .slice(0, MAX_SYNTHESIS_FINDINGS) + .map(({ f }) => f); + + // Organize findings by their own angle (provenance-aware) + const evidenceByAngle = new Map(); + for (const finding of ranked) { + const angle = finding.angle ?? "general"; + if (!evidenceByAngle.has(angle)) evidenceByAngle.set(angle, []); + evidenceByAngle.get(angle)!.push(finding); + } + + let evidenceText = `## Research Question\n${question}\n\n`; + evidenceText += `## Overview\n- Rounds of research: ${rounds.length}\n`; + evidenceText += `- Total searches executed: ${totalSearches}\n`; + evidenceText += `- Total pages analyzed: ${totalPages}\n`; + evidenceText += `- Key findings extracted: ${allFindings.length} (${ranked.length} passed dedup/quality filter)\n\n`; + + // Build evidence grouped by angle with reference IDs + for (const [angle, findings] of Array.from(evidenceByAngle)) { + if (findings.length === 0) continue; + evidenceText += `## Angle: ${angle}\n\n`; + for (const finding of findings) { + // Get reference IDs for this finding's sources + const refs = finding.sources + .map((url) => seenUrls.get(url)) + .filter((r): r is Reference => !!r) + .map((r) => `[${r.id}]`); + + const avgAuth = + finding.avgSourceAuthority !== undefined + ? ` | Avg Authority: ${(finding.avgSourceAuthority * 100).toFixed(0)}%` + : ""; + const corr = + finding.corroborationScore !== undefined + ? ` | Corroboration: ${(finding.corroborationScore * 100).toFixed(0)}%` + : ""; + const bestAuthStr = + finding.bestSourceAuthority !== undefined + ? ` | Best Source: ${(finding.bestSourceAuthority * 100).toFixed(0)}%` + : ""; + + evidenceText += `### ${finding.title}\n`; + evidenceText += `**Confidence:** ${finding.confidence}${avgAuth}${corr}${bestAuthStr}\n`; + if (refs.length > 0) { + evidenceText += `**Sources:** ${refs.join(", ")}\n`; + } + evidenceText += `${finding.summary}\n\n`; + if (finding.keyQuotes.length > 0) { + evidenceText += `> ${finding.keyQuotes[0]}\n\n`; + } + } + } + + // Include reference metadata for the LLM to build proper citations + evidenceText += `## Reference Metadata\n\n`; + for (const [, ref] of seenUrls) { + evidenceText += `[${ref.id}] ${ref.title} (${ref.domain}, authority: ${(ref.authorityScore * 100).toFixed(0)}%) — ${ref.url}\n`; + } + + return { evidenceText, referenceMap: seenUrls }; +} + +/* ── Main Synthesis ──────────────────────────────────────────────── */ + +/** + * Synthesize a research report from all rounds. + * Returns both the formatted report and the full bibliography. + */ +export async function synthesizeReport( + question: string, + rounds: ResearchRound[], + config: ResearchConfig, + cwd: string, + signal?: AbortSignal, +): Promise { + const audience = config.audience ?? "general"; + const { evidenceText, referenceMap } = buildEvidenceText(question, rounds); + + const formatInstruction = + config.format === "structured" + ? "Structured report with numbered sections, clear hierarchies, and data tables where appropriate." + : "Well-formatted markdown report with ## headings, bullet points, and inline numbered citations like [1]."; + + const taskPrompt = `Synthesize the following research findings into a comprehensive, well-structured report. + +${evidenceText} + +Write a thorough report that answers the original question: "${question}" + +Format: ${formatInstruction} +Audience: ${audience} + +Remember to use numbered citations like [1], [2] and include a ## References section at the end.`; + + const result = await runAnalysisAgent( + buildSynthesisSystem(audience), + taskPrompt, + cwd, + 120_000, + undefined, + signal, + ); + + if (result.success && result.text) { + // Build bibliography section + const bibSection = buildBibliography(referenceMap); + + let report = result.text; + + // ── Citation integrity ───────────────────────────────────────── + // 1. Strip any references section the LLM wrote and replace it with + // the authoritative bibliography (built from real scraped sources). + // 2. Remove inline [n] citations that point at IDs outside the + // bibliography (hallucinated numbers), so every citation resolves. + report = report.replace( + /^#+\s*references\s*$/gim, + "\n## END_OF_REPORT_MARKER", + ); + const markerIdx = report.indexOf("## END_OF_REPORT_MARKER"); + if (markerIdx !== -1) { + report = report.slice(0, markerIdx).trimEnd(); + } + + const maxRefId = Math.max( + 0, + ...Array.from(referenceMap.values()).map((r) => r.id), + ); + report = report.replace(/\[(\d+)\]/g, (match, id: string) => { + const num = parseInt(id, 10); + return num >= 1 && num <= maxRefId ? match : ""; + }); + + report = report.trimEnd() + `\n\n${bibSection}`; + + return { report, references: Array.from(referenceMap.values()) }; + } + + // Fallback: generate a simple structured report + const fallbackReport = generateFallbackReport( + question, + rounds, + referenceMap, + audience, + ); + return { + report: fallbackReport + `\n\n${buildBibliography(referenceMap)}`, + references: Array.from(referenceMap.values()), + }; +} + +/* ── Bibliography Builder ────────────────────────────────────────── */ + +/** + * Build a structured ## References section from the reference map. + */ +function buildBibliography(referenceMap: Map): string { + if (referenceMap.size === 0) return "## References\n\nNo sources cited."; + + const refs = Array.from(referenceMap.values()).sort((a, b) => a.id - b.id); + const lines: string[] = ["## References\n"]; + for (const ref of refs) { + const authIcon = + ref.authorityScore >= 0.8 ? "⭐" : ref.authorityScore >= 0.5 ? "✓" : "○"; + lines.push( + `[${ref.id}] ${authIcon} **${ref.title}** — ${ref.domain} (${ref.url}) — accessed ${ref.accessedAt}`, + ); + } + + return lines.join("\n"); +} + +/* ── Fallback Report ─────────────────────────────────────────────── */ + +/** + * Fallback report when the LLM synthesis fails. + * Produces a clean, structured report from the evidence. + */ +function generateFallbackReport( + question: string, + rounds: ResearchRound[], + referenceMap: Map, + _audience: string, +): string { + const lines: string[] = []; + const allFindings = rounds.flatMap((r) => r.findings); + + // ── TL;DR ── + lines.push(`# Research Report: ${question}`); + lines.push(""); + + const highConfFindings = allFindings.filter((f) => f.confidence === "high"); + const totalHigh = highConfFindings.length; + const total = allFindings.length; + + lines.push("## TL;DR"); + lines.push(""); + if (highConfFindings.length > 0) { + lines.push( + `Based on analysis of ${total} findings across ${rounds.length} research round(s), ` + + `${totalHigh} high-confidence conclusions were identified. ` + + `${highConfFindings[0].title}: ${highConfFindings[0].summary}`, + ); + } else { + lines.push( + `This report covers findings from ${rounds.length} research round(s) exploring "${question}". ` + + `${total} findings were extracted, with varying levels of confidence.`, + ); + } + lines.push(""); + + // ── Executive Summary ── + lines.push("## Executive Summary"); + lines.push(""); + lines.push( + `This report synthesizes findings from ${rounds.length} research round(s), ` + + `${rounds.reduce((s, r) => s + r.queries.length, 0)} search queries, ` + + `and ${rounds.reduce((s, r) => s + r.results.length, 0)} sources.`, + ); + lines.push(""); + + // ── Key Findings (tiered) ── + if (allFindings.length > 0) { + lines.push("## Key Findings"); + lines.push(""); + + // High confidence first + const highConf = allFindings.filter((f) => f.confidence === "high"); + if (highConf.length > 0) { + lines.push("### High Confidence"); + for (const finding of highConf) { + const refs = finding.sources + .map((url) => referenceMap.get(url)) + .filter((r): r is Reference => !!r) + .map((r) => `[${r.id}]`); + lines.push( + `- **${finding.title}** ${refs.length > 0 ? refs.join("") : ""}`, + ); + lines.push(` - ${finding.summary}`); + } + lines.push(""); + } + + // Medium confidence + const medConf = allFindings.filter((f) => f.confidence === "medium"); + if (medConf.length > 0) { + lines.push("### Moderate Confidence"); + for (const finding of medConf) { + const refs = finding.sources + .map((url) => referenceMap.get(url)) + .filter((r): r is Reference => !!r) + .map((r) => `[${r.id}]`); + lines.push( + `- **${finding.title}** ${refs.length > 0 ? refs.join("") : ""}`, + ); + lines.push(` - ${finding.summary}`); + } + lines.push(""); + } + + // Low confidence + const lowConf = allFindings.filter((f) => f.confidence === "low"); + if (lowConf.length > 0) { + lines.push("### Lower Confidence (Needs Further Research)"); + for (const finding of lowConf) { + const refs = finding.sources + .map((url) => referenceMap.get(url)) + .filter((r): r is Reference => !!r) + .map((r) => `[${r.id}]`); + lines.push( + `- **${finding.title}** ${refs.length > 0 ? refs.join("") : ""}`, + ); + lines.push(` - ${finding.summary}`); + } + lines.push(""); + } + + // ── Detailed Analysis ── + lines.push("## Detailed Analysis"); + lines.push(""); + + const byAngle = new Map(); + for (const round of rounds) { + for (const f of round.findings) { + const angle = f.angle ?? round.queries[0]?.angle ?? "general"; + if (!byAngle.has(angle)) byAngle.set(angle, []); + byAngle.get(angle)!.push(f); + } + } + + for (const [angle, findings] of byAngle) { + lines.push(`### ${angle.charAt(0).toUpperCase() + angle.slice(1)}`); + lines.push(""); + for (const f of findings) { + const corrStr = + f.corroborationScore !== undefined + ? ` (corroboration: ${(f.corroborationScore * 100).toFixed(0)}%)` + : ""; + lines.push(`**${f.title}** — *${f.confidence} confidence${corrStr}*`); + lines.push(""); + lines.push(f.summary); + lines.push(""); + if (f.keyQuotes.length > 0) { + lines.push(`> ${f.keyQuotes[0]}`); + lines.push(""); + } + } + } + + // ── Limitations ── + const lowConfCount = allFindings.filter( + (f) => f.confidence === "low", + ).length; + const noCorr = allFindings.filter( + (f) => (f.corroborationScore ?? 0) < 0.3, + ).length; + + lines.push("## Limitations & Knowledge Gaps"); + lines.push(""); + if (lowConfCount > 0) { + lines.push( + `- **${lowConfCount} of ${allFindings.length} findings** have low confidence, indicating limited or conflicting evidence.`, + ); + } + if (noCorr > 0) { + lines.push( + `- **${noCorr} findings** lack corroboration from multiple independent sources.`, + ); + } + lines.push( + "- This research relied on web search results; some relevant sources may not be indexed or accessible.", + ); + lines.push( + "- Findings are dependent on search engine ranking and the quality of indexed content.", + ); + lines.push(""); + + // ── Conclusion ── + lines.push("## Conclusion"); + lines.push(""); + if (highConf.length > 0) { + lines.push( + `The research identified ${highConf.length} high-confidence finding(s) and ${medConf.length} moderately-supported finding(s). ` + + `The strongest evidence relates to: ${highConf.map((f) => f.title).join(", ")}.`, + ); + } else { + lines.push( + "The research surfaced relevant information but with limited high-confidence evidence. Further investigation is recommended for the identified knowledge gaps.", + ); + } + lines.push(""); + } + + // ── Methodology ── + lines.push(`*Report prepared for: ${_audience} audience*`); + lines.push(""); + + lines.push("## Methodology"); + lines.push(""); + for (const round of rounds) { + const failedSearches = + round.failedSearches ?? round.queries.length - round.successfulSearches; + lines.push(`### Round ${round.round}`); + lines.push( + `Queries: ${round.queries.map((q) => `"${q.query}" [${q.angle}]`).join(", ")}`, + ); + lines.push(`Pages scraped: ${round.results.length}`); + lines.push(`Findings extracted: ${round.findings.length}`); + if (failedSearches > 0) { + lines.push(`Searches failed: ${failedSearches}`); + } + lines.push(""); + } + + return lines.join("\n"); +} diff --git a/src/research.ts b/src/research.ts new file mode 100644 index 0000000..6a46a8d --- /dev/null +++ b/src/research.ts @@ -0,0 +1,468 @@ +/** + * Deep Research — Core research orchestration + * + * Manages the multi-round deep research process: + * 1. Decompose the question into sub-questions (when depth > 1) + * 2. Generate initial search queries (per sub-question for better diversity) + * 3. Execute all queries in parallel via Firecrawl + * 4. Analyze results and extract findings + * 5. Compute corroboration scores + * 6. Generate follow-up queries for gaps + * 7. Iterate for depth rounds + * 8. Synthesize final report with numbered references + * + * Widget and progress callback patterns borrowed from ralpi's executor. + */ +import type { ExtensionContext } from "@oh-my-pi/pi-coding-agent"; +import type { + Finding, + ResearchConfig, + EnrichedSearchResult, + ResearchRound, + ResearchReport, +} from "./types"; +import type { SynthesisResult } from "./report"; +import { searchWeb, isNearDuplicateTitle } from "./firecrawl"; +import { + generateQueries, + generateFollowUpQueries, + analyzeResults, + computeCorroboration, + decomposeQuestion, +} from "./queries"; +import { synthesizeReport } from "./report"; + +/** Progress callback for UI updates */ +export type ResearchProgress = (update: { + phase: + | "decomposing" + | "generating_queries" + | "searching" + | "analyzing" + | "synthesizing" + | "complete"; + round?: number; + totalRounds?: number; + message: string; + detail?: string; + fraction?: number; // 0-1 +}) => void; + +// ── Round-Robin Parallel Execution ────────────────────────────────── + +/** + * Maximum concurrent Firecrawl search requests. + * Prevents rate limiting while still parallelizing queries. + */ +const MAX_SEARCH_CONCURRENT = 3; + +/** + * Maximum concurrent analysis agent sessions. + */ +const MAX_ANALYSIS_CONCURRENT = 2; + +/** + * Minimum findings per round before we consider early stopping. + * If we're getting very few new findings, saturation is near. + */ +const SATURATION_THRESHOLD = 0.15; // < 15% new findings = likely saturated + +/** + * Bounded-concurrency parallel execution with round-robin slot assignment. + * + * Similar to ralpi's ModelRoundRobin: with N concurrent slots, items are + * assigned to free slots in FIFO order. When a slot finishes, the next + * item in the queue is assigned to it. + * + * This ensures even load distribution and avoids bursty concurrency. + */ +async function boundedConcurrency( + items: T[], + maxConcurrent: number, + mapper: (item: T, index: number) => Promise, +): Promise { + const results: R[] = new Array(items.length); + let nextIndex = 0; + + async function worker(): Promise { + while (true) { + const currentIndex = nextIndex++; + if (currentIndex >= items.length) return; + results[currentIndex] = await mapper(items[currentIndex], currentIndex); + } + } + + const numWorkers = Math.min(maxConcurrent, items.length); + const workers = Array.from({ length: numWorkers }, () => worker()); + await Promise.all(workers); + return results; +} + +/** + * Assess whether the research is reaching information saturation. + */ +function assessSaturation( + previousRound: ResearchRound | undefined, + currentRound: ResearchRound, +): number { + if (!previousRound || previousRound.findings.length === 0) return 0; + + const prevUrls = new Set(previousRound.results.map((r) => r.url)); + const newUrls = currentRound.results.filter( + (r) => !prevUrls.has(r.url), + ).length; + const totalUrls = currentRound.results.length; + const newRatio = totalUrls > 0 ? newUrls / totalUrls : 0; + + // Also check finding novelty + const prevFindingTitles = new Set( + previousRound.findings.map((f) => f.title.toLowerCase()), + ); + const newFindings = currentRound.findings.filter( + (f) => !prevFindingTitles.has(f.title.toLowerCase()), + ).length; + const totalFindings = currentRound.findings.length; + const findingNovelty = totalFindings > 0 ? newFindings / totalFindings : 0; + + // Weight: URL novelty (40%) + finding novelty (60%) + return newRatio * 0.4 + findingNovelty * 0.6; +} + +/** + * Run a complete deep research session. + */ +export async function runDeepResearch( + config: ResearchConfig, + ctx: ExtensionContext, + onProgress: ResearchProgress, + signal?: AbortSignal, +): Promise { + const startTime = Date.now(); + const rounds: ResearchRound[] = []; + let totalSearches = 0; + let totalPages = 0; + let subQuestions: string[] = []; + + // ── Phase: Decompose question into sub-questions ──────────────── + + if (config.depth > 1) { + onProgress({ + phase: "decomposing", + round: 1, + totalRounds: config.depth, + message: "Decomposing research question into sub-topics...", + fraction: 0, + }); + + if (signal?.aborted) throw new Error("Research cancelled"); + + subQuestions = await decomposeQuestion(config.question, ctx.cwd, signal); + } + + // ── Phase: Generate initial queries ───────────────────────────── + + onProgress({ + phase: "generating_queries", + round: 1, + totalRounds: config.depth, + message: + subQuestions.length > 0 + ? `Generating queries across ${subQuestions.length} sub-topics...` + : "Generating initial search queries...", + fraction: 0.05, + }); + + if (signal?.aborted) throw new Error("Research cancelled"); + + const queries = await generateQueries( + config.question, + config.breadth, + ctx.cwd, + signal, + subQuestions.length > 0 ? subQuestions : undefined, + ); + + if (queries.length === 0) { + throw new Error("Failed to generate any search queries"); + } + + // ── Execute rounds ─────────────────────────────────────────────── + + for (let round = 1; round <= config.depth; round++) { + if (signal?.aborted) throw new Error("Research cancelled"); + + const isFirstRound = round === 1; + const currentQueries = isFirstRound + ? queries + : await generateFollowUpQueries( + config.question, + rounds, + config.breadth, + ctx.cwd, + signal, + ); + + if (!currentQueries || currentQueries.length === 0) { + // No follow-up queries to generate — stop here + break; + } + + // ── Search phase (parallel with round-robin) ──────────────────── + + onProgress({ + phase: "searching", + round, + totalRounds: config.depth, + message: `Searching ${currentQueries.length} queries in parallel...`, + fraction: 0.25, + }); + + if (signal?.aborted) throw new Error("Research cancelled"); + + // Run searches in parallel using round-robin bounded concurrency. + // Each mapper call runs independently; failures are caught per-query. + // Results keep their originating query index for later grouping. + const searchResultsArrays: (EnrichedSearchResult[] | null)[] = + await boundedConcurrency( + currentQueries, + MAX_SEARCH_CONCURRENT, + async (q, i) => { + onProgress({ + phase: "searching", + round, + totalRounds: config.depth, + message: `Searching: "${q.query.slice(0, 60)}..."`, + detail: q.rationale, + fraction: 0.25 + (i / currentQueries.length) * 0.25, + }); + + try { + return await searchWeb(q.query, 5, signal); + } catch (error) { + const errorMsg = + error instanceof Error ? error.message : String(error); + onProgress({ + phase: "searching", + round, + totalRounds: config.depth, + message: `Search failed: ${errorMsg.slice(0, 80)}`, + fraction: 0.25 + ((i + 1) / currentQueries.length) * 0.25, + }); + return null; + } + }, + ); + + const successfulSearches = searchResultsArrays.filter( + (r): r is EnrichedSearchResult[] => r !== null, + ).length; + const failedSearches = currentQueries.length - successfulSearches; + + totalSearches += currentQueries.length; + + // Track which URLs were independently surfaced by MULTIPLE different + // queries. This is the corroboration signal: a source found by + // several independent searches is stronger evidence than one found + // by a single query. + const urlQueryCounts = new Map(); + searchResultsArrays.forEach((results, queryIndex) => { + if (!results) return; + const queryUrls = new Set(results.map((r) => r.url)); + for (const url of queryUrls) { + urlQueryCounts.set(url, (urlQueryCounts.get(url) ?? 0) + 1); + } + // (queryIndex is unused beyond the closure; kept for clarity) + void queryIndex; + }); + + // ── Per-query result collection ────────────────────────────────── + // Deduplicate within each query's results by URL (prefer higher + // authority) AND by near-identical title (catches syndicated copies + // of the same article under different URLs). + const resultsByQuery: EnrichedSearchResult[][] = currentQueries.map( + () => [], + ); + + searchResultsArrays.forEach((results, queryIndex) => { + if (!results) return; + const seenUrls = new Set(); + const seenTitles: string[] = []; + for (const r of results) { + if (seenUrls.has(r.url)) continue; + seenUrls.add(r.url); + + // Skip syndicated duplicates (same article, different URL) + if ( + r.title && + seenTitles.some((t) => isNearDuplicateTitle(t, r.title)) + ) { + continue; + } + if (r.title) seenTitles.push(r.title); + resultsByQuery[queryIndex].push(r); + } + }); + + // Global URL dedup across queries: a URL found by multiple queries + // stays attached to the FIRST query that surfaced it (most likely + // the most relevant one). + const globalSeen = new Set(); + for (const list of resultsByQuery) { + for (let i = list.length - 1; i >= 0; i--) { + if (globalSeen.has(list[i].url)) { + list.splice(i, 1); + } else { + globalSeen.add(list[i].url); + } + } + } + + const uniqueResults = resultsByQuery.flat(); + totalPages += uniqueResults.length; + + // ── Analyze phase (parallel with round-robin) ────────────────── + + onProgress({ + phase: "analyzing", + round, + totalRounds: config.depth, + message: `Analyzing ${uniqueResults.length} search results in parallel...`, + fraction: 0.6, + }); + + if (signal?.aborted) throw new Error("Research cancelled"); + + // Build query-result pairs for parallel analysis. + // Each query is analyzed with the results IT actually produced, + // so findings stay coherent with the query's intent and angle. + const analysisTasks: Array<{ + query: (typeof currentQueries)[number]; + results: EnrichedSearchResult[]; + index: number; + }> = []; + + for (let i = 0; i < currentQueries.length; i++) { + const queryResults = resultsByQuery[i]; + if (!queryResults || queryResults.length === 0) continue; + + analysisTasks.push({ + query: currentQueries[i], + results: queryResults, + index: i, + }); + } + + // Run analyses in parallel using round-robin bounded concurrency + const findingsArrays: Finding[][] = await boundedConcurrency( + analysisTasks, + MAX_ANALYSIS_CONCURRENT, + async (task) => { + onProgress({ + phase: "analyzing", + round, + totalRounds: config.depth, + message: `Analyzing: "${task.query.query.slice(0, 40)}..."`, + fraction: + 0.6 + (task.index / Math.max(analysisTasks.length, 1)) * 0.2, + }); + + try { + return await analyzeResults( + task.query.query, + task.results, + ctx.cwd, + signal, + task.query.angle, + ); + } catch { + // Analysis failure shouldn't crash the round + return []; + } + }, + ); + + // Flatten all findings + const allFindings: ResearchRound["findings"] = findingsArrays.flat(); + + // ── Corroboration pass ──────────────────────────────────────── + // Cross-reference findings to compute corroboration scores. + // Corroboration = fraction of a finding's sources that were + // independently surfaced by multiple different search queries. + const corroboratedFindings = computeCorroboration( + allFindings, + urlQueryCounts, + ); + + // Record this round + const followUpTopics = corroboratedFindings + .filter( + (f: Finding) => + f.confidence === "low" && (f.corroborationScore ?? 0) < 0.5, + ) + .map((f: Finding) => f.title); + + rounds.push({ + round, + queries: currentQueries, + results: uniqueResults, + findings: corroboratedFindings, + followUpTopics, + successfulSearches, + failedSearches, + }); + + // ── Adaptive depth: check for saturation ────────────────────── + if (round > 1 && round < config.depth) { + const saturation = assessSaturation( + rounds[rounds.length - 2], + rounds[rounds.length - 1], + ); + if (saturation < SATURATION_THRESHOLD) { + onProgress({ + phase: "synthesizing", + message: `Information saturation reached (${(saturation * 100).toFixed(0)}% novelty) — synthesizing early`, + fraction: 0.85, + }); + break; + } + } + } + + // ── Synthesis phase ─────────────────────────────────────────────── + + onProgress({ + phase: "synthesizing", + message: "Synthesizing research into final report...", + fraction: 0.9, + }); + + if (signal?.aborted) throw new Error("Research cancelled"); + + const synthesisResult: SynthesisResult = await synthesizeReport( + config.question, + rounds, + config, + ctx.cwd, + signal, + ); + const finalReport = synthesisResult.report; + const references = synthesisResult.references; + + const durationMs = Date.now() - startTime; + + onProgress({ + phase: "complete", + message: "Research complete!", + fraction: 1.0, + }); + + return { + question: config.question, + rounds, + finalReport, + totalSearches, + totalPagesScraped: totalPages, + durationMs, + references, + }; +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..e880d31 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,106 @@ +/** + * Deep Research — type definitions + */ + +/** Content type classification for a source */ +export type ContentType = + | "documentation" + | "paper" + | "news" + | "blog" + | "forum" + | "official" + | "other"; + +/** A single search result from Firecrawl */ +export interface SearchResult { + title: string; + url: string; + description: string; + markdown: string; +} + +/** Enriched search result with source authority metadata */ +export interface EnrichedSearchResult extends SearchResult { + domain: string; + authorityScore: number; // 0.0 – 1.0 + publishedDate: Date | null; + contentType: ContentType; +} + +/** A finding extracted from search results by an analysis agent */ +export interface Finding { + title: string; + summary: string; + sources: string[]; + keyQuotes: string[]; + confidence: "high" | "medium" | "low"; + /** The search query this finding was extracted under (provenance) */ + query?: string; + /** The research angle of the originating query (provenance) */ + angle?: string; + /** 0.0 – 1.0: how many independent sources support this finding */ + corroborationScore?: number; + /** Authority score of the best source supporting this finding */ + bestSourceAuthority?: number; + /** Average authority score across all sources */ + avgSourceAuthority?: number; +} + +/** A numbered reference with full metadata */ +export interface Reference { + id: number; + url: string; + title: string; + domain: string; + authorityScore: number; + accessedAt: string; // ISO date string +} + +/** A generated search query with its intent/rationale */ +export interface SearchQuery { + query: string; + rationale: string; + angle: string; +} + +/** Output from one research round */ +export interface ResearchRound { + round: number; + queries: SearchQuery[]; + results: EnrichedSearchResult[]; + findings: Finding[]; + /** Any follow-up questions/angles the analysis suggests */ + followUpTopics: string[]; + /** Number of search queries that actually returned data (non-empty) */ + successfulSearches: number; + /** Number of search queries that failed entirely */ + failedSearches: number; +} + +/** Target audience expertise level */ +export type Audience = "expert" | "general" | "executive"; + +/** Configuration for a research session */ +export interface ResearchConfig { + question: string; + depth: number; // 1-3 rounds + breadth: number; // queries per round (1-5) + format: "markdown" | "structured"; + audience?: Audience; + /** Focus on specific research angles only (empty = all angles) */ + focus?: string[]; + /** Show the research methodology section in the report */ + showMethodology?: boolean; +} + +/** Final research report */ +export interface ResearchReport { + question: string; + rounds: ResearchRound[]; + finalReport: string; + totalSearches: number; + totalPagesScraped: number; + durationMs: number; + references: Reference[]; +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..7e287c7 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "node", + "lib": ["ES2022"], + "noEmit": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true + }, + "include": ["index.ts", "src/**/*"], + "exclude": ["node_modules", "dist"] +}